Support original-detail metadata on MCP image outputs (#17714)

## Summary
- honor `_meta["codex/imageDetail"] == "original"` on MCP image content
and map it to `detail: "original"` where supported
- strip that detail back out when the active model does not support
original-detail image inputs
- update code-mode `image(...)` to accept individual MCP image blocks
- teach `js_repl` / `codex.emitImage(...)` to preserve the same hint
from raw MCP image outputs
- document the new `_meta` contract and add generic RMCP-backed coverage
across protocol, core, code-mode, and js_repl paths
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-04-15 14:43:33 -07:00
committed by GitHub
parent 17d94bd1e3
commit 9e2fc31854
20 changed files with 905 additions and 368 deletions
+9
View File
@@ -18,6 +18,8 @@ use tokio_util::sync::CancellationToken;
use crate::codex::Session;
use crate::codex::TurnContext;
use crate::function_tool::FunctionCallError;
use crate::original_image_detail::can_request_original_image_detail;
use crate::original_image_detail::sanitize_original_image_detail as sanitize_image_detail_items;
use crate::tools::ToolRouter;
use crate::tools::context::FunctionToolOutput;
use crate::tools::context::SharedTurnDiffTracker;
@@ -160,12 +162,14 @@ pub(super) async fn handle_runtime_response(
match response {
RuntimeResponse::Yielded { content_items, .. } => {
let mut content_items = into_function_call_output_content_items(content_items);
sanitize_runtime_image_detail(exec.turn.as_ref(), &mut content_items);
content_items = truncate_code_mode_result(content_items, max_output_tokens);
prepend_script_status(&mut content_items, &script_status, started_at.elapsed());
Ok(FunctionToolOutput::from_content(content_items, Some(true)))
}
RuntimeResponse::Terminated { content_items, .. } => {
let mut content_items = into_function_call_output_content_items(content_items);
sanitize_runtime_image_detail(exec.turn.as_ref(), &mut content_items);
content_items = truncate_code_mode_result(content_items, max_output_tokens);
prepend_script_status(&mut content_items, &script_status, started_at.elapsed());
Ok(FunctionToolOutput::from_content(content_items, Some(true)))
@@ -177,6 +181,7 @@ pub(super) async fn handle_runtime_response(
..
} => {
let mut content_items = into_function_call_output_content_items(content_items);
sanitize_runtime_image_detail(exec.turn.as_ref(), &mut content_items);
exec.session
.services
.code_mode_service
@@ -198,6 +203,10 @@ pub(super) async fn handle_runtime_response(
}
}
fn sanitize_runtime_image_detail(turn: &TurnContext, items: &mut [FunctionCallOutputContentItem]) {
sanitize_image_detail_items(can_request_original_image_detail(&turn.model_info), items);
}
fn format_script_status(response: &RuntimeResponse) -> String {
match response {
RuntimeResponse::Yielded { cell_id, .. } => {
+6
View File
@@ -1,5 +1,6 @@
use crate::codex::Session;
use crate::codex::TurnContext;
use crate::original_image_detail::sanitize_original_image_detail;
use crate::tools::TELEMETRY_PREVIEW_MAX_BYTES;
use crate::tools::TELEMETRY_PREVIEW_MAX_LINES;
use crate::tools::TELEMETRY_PREVIEW_TRUNCATION_NOTICE;
@@ -122,6 +123,7 @@ impl ToolOutput for CallToolResult {
pub struct McpToolOutput {
pub result: CallToolResult,
pub wall_time: Duration,
pub original_image_detail_supported: bool,
}
impl ToolOutput for McpToolOutput {
@@ -155,6 +157,10 @@ impl ToolOutput for McpToolOutput {
impl McpToolOutput {
fn response_payload(&self) -> FunctionCallOutputPayload {
let mut payload = self.result.as_function_call_output_payload();
if let Some(items) = payload.content_items_mut() {
sanitize_original_image_detail(self.original_image_detail_supported, items);
}
let wall_time_seconds = self.wall_time.as_secs_f64();
let header = format!("Wall time: {wall_time_seconds:.4} seconds\nOutput:");
+3
View File
@@ -98,6 +98,7 @@ fn mcp_tool_output_response_item_includes_wall_time() {
meta: None,
},
wall_time: std::time::Duration::from_millis(1250),
original_image_detail_supported: false,
};
let response = output.to_response_item(
@@ -149,6 +150,7 @@ fn mcp_tool_output_response_item_preserves_content_items() {
meta: None,
},
wall_time: std::time::Duration::from_millis(500),
original_image_detail_supported: false,
};
let response = output.to_response_item(
@@ -201,6 +203,7 @@ fn mcp_tool_output_code_mode_result_stays_raw_call_tool_result() {
meta: None,
},
wall_time: std::time::Duration::from_millis(1250),
original_image_detail_supported: false,
};
let result = output.code_mode_result(&ToolPayload::Mcp {
+2
View File
@@ -3,6 +3,7 @@ use std::time::Instant;
use crate::function_tool::FunctionCallError;
use crate::mcp_tool_call::handle_mcp_tool_call;
use crate::original_image_detail::can_request_original_image_detail;
use crate::tools::context::McpToolOutput;
use crate::tools::context::ToolInvocation;
use crate::tools::context::ToolPayload;
@@ -56,6 +57,7 @@ impl ToolHandler for McpHandler {
Ok(McpToolOutput {
result,
wall_time: started.elapsed(),
original_image_detail_supported: can_request_original_image_detail(&turn.model_info),
})
}
}
@@ -1330,6 +1330,13 @@ function normalizeMcpImageData(data, mimeType) {
return `data:${normalizedMimeType};base64,${data}`;
}
function parseMcpImageDetail(meta) {
if (!isPlainObject(meta) || meta["codex/imageDetail"] !== "original") {
return undefined;
}
return "original";
}
function parseMcpToolResult(result) {
if (typeof result === "string") {
return { images: [], textCount: result.length > 0 ? 1 : 0 };
@@ -1362,6 +1369,7 @@ function parseMcpToolResult(result) {
if (item.type === "image") {
images.push({
image_url: normalizeMcpImageData(item.data, item.mimeType ?? item.mime_type),
detail: parseMcpImageDetail(item._meta),
});
continue;
}