mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
Support original-detail metadata on MCP image outputs (#17714)
## Summary - honor `_meta["codex/imageDetail"] == "original"` on MCP image content and map it to `detail: "original"` where supported - strip that detail back out when the active model does not support original-detail image inputs - update code-mode `image(...)` to accept individual MCP image blocks - teach `js_repl` / `codex.emitImage(...)` to preserve the same hint from raw MCP image outputs - document the new `_meta` contract and add generic RMCP-backed coverage across protocol, core, code-mode, and js_repl paths
This commit is contained in:
@@ -18,6 +18,8 @@ use tokio_util::sync::CancellationToken;
|
||||
use crate::codex::Session;
|
||||
use crate::codex::TurnContext;
|
||||
use crate::function_tool::FunctionCallError;
|
||||
use crate::original_image_detail::can_request_original_image_detail;
|
||||
use crate::original_image_detail::sanitize_original_image_detail as sanitize_image_detail_items;
|
||||
use crate::tools::ToolRouter;
|
||||
use crate::tools::context::FunctionToolOutput;
|
||||
use crate::tools::context::SharedTurnDiffTracker;
|
||||
@@ -160,12 +162,14 @@ pub(super) async fn handle_runtime_response(
|
||||
match response {
|
||||
RuntimeResponse::Yielded { content_items, .. } => {
|
||||
let mut content_items = into_function_call_output_content_items(content_items);
|
||||
sanitize_runtime_image_detail(exec.turn.as_ref(), &mut content_items);
|
||||
content_items = truncate_code_mode_result(content_items, max_output_tokens);
|
||||
prepend_script_status(&mut content_items, &script_status, started_at.elapsed());
|
||||
Ok(FunctionToolOutput::from_content(content_items, Some(true)))
|
||||
}
|
||||
RuntimeResponse::Terminated { content_items, .. } => {
|
||||
let mut content_items = into_function_call_output_content_items(content_items);
|
||||
sanitize_runtime_image_detail(exec.turn.as_ref(), &mut content_items);
|
||||
content_items = truncate_code_mode_result(content_items, max_output_tokens);
|
||||
prepend_script_status(&mut content_items, &script_status, started_at.elapsed());
|
||||
Ok(FunctionToolOutput::from_content(content_items, Some(true)))
|
||||
@@ -177,6 +181,7 @@ pub(super) async fn handle_runtime_response(
|
||||
..
|
||||
} => {
|
||||
let mut content_items = into_function_call_output_content_items(content_items);
|
||||
sanitize_runtime_image_detail(exec.turn.as_ref(), &mut content_items);
|
||||
exec.session
|
||||
.services
|
||||
.code_mode_service
|
||||
@@ -198,6 +203,10 @@ pub(super) async fn handle_runtime_response(
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_runtime_image_detail(turn: &TurnContext, items: &mut [FunctionCallOutputContentItem]) {
|
||||
sanitize_image_detail_items(can_request_original_image_detail(&turn.model_info), items);
|
||||
}
|
||||
|
||||
fn format_script_status(response: &RuntimeResponse) -> String {
|
||||
match response {
|
||||
RuntimeResponse::Yielded { cell_id, .. } => {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use crate::codex::Session;
|
||||
use crate::codex::TurnContext;
|
||||
use crate::original_image_detail::sanitize_original_image_detail;
|
||||
use crate::tools::TELEMETRY_PREVIEW_MAX_BYTES;
|
||||
use crate::tools::TELEMETRY_PREVIEW_MAX_LINES;
|
||||
use crate::tools::TELEMETRY_PREVIEW_TRUNCATION_NOTICE;
|
||||
@@ -122,6 +123,7 @@ impl ToolOutput for CallToolResult {
|
||||
pub struct McpToolOutput {
|
||||
pub result: CallToolResult,
|
||||
pub wall_time: Duration,
|
||||
pub original_image_detail_supported: bool,
|
||||
}
|
||||
|
||||
impl ToolOutput for McpToolOutput {
|
||||
@@ -155,6 +157,10 @@ impl ToolOutput for McpToolOutput {
|
||||
impl McpToolOutput {
|
||||
fn response_payload(&self) -> FunctionCallOutputPayload {
|
||||
let mut payload = self.result.as_function_call_output_payload();
|
||||
if let Some(items) = payload.content_items_mut() {
|
||||
sanitize_original_image_detail(self.original_image_detail_supported, items);
|
||||
}
|
||||
|
||||
let wall_time_seconds = self.wall_time.as_secs_f64();
|
||||
let header = format!("Wall time: {wall_time_seconds:.4} seconds\nOutput:");
|
||||
|
||||
|
||||
@@ -98,6 +98,7 @@ fn mcp_tool_output_response_item_includes_wall_time() {
|
||||
meta: None,
|
||||
},
|
||||
wall_time: std::time::Duration::from_millis(1250),
|
||||
original_image_detail_supported: false,
|
||||
};
|
||||
|
||||
let response = output.to_response_item(
|
||||
@@ -149,6 +150,7 @@ fn mcp_tool_output_response_item_preserves_content_items() {
|
||||
meta: None,
|
||||
},
|
||||
wall_time: std::time::Duration::from_millis(500),
|
||||
original_image_detail_supported: false,
|
||||
};
|
||||
|
||||
let response = output.to_response_item(
|
||||
@@ -201,6 +203,7 @@ fn mcp_tool_output_code_mode_result_stays_raw_call_tool_result() {
|
||||
meta: None,
|
||||
},
|
||||
wall_time: std::time::Duration::from_millis(1250),
|
||||
original_image_detail_supported: false,
|
||||
};
|
||||
|
||||
let result = output.code_mode_result(&ToolPayload::Mcp {
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::time::Instant;
|
||||
|
||||
use crate::function_tool::FunctionCallError;
|
||||
use crate::mcp_tool_call::handle_mcp_tool_call;
|
||||
use crate::original_image_detail::can_request_original_image_detail;
|
||||
use crate::tools::context::McpToolOutput;
|
||||
use crate::tools::context::ToolInvocation;
|
||||
use crate::tools::context::ToolPayload;
|
||||
@@ -56,6 +57,7 @@ impl ToolHandler for McpHandler {
|
||||
Ok(McpToolOutput {
|
||||
result,
|
||||
wall_time: started.elapsed(),
|
||||
original_image_detail_supported: can_request_original_image_detail(&turn.model_info),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1330,6 +1330,13 @@ function normalizeMcpImageData(data, mimeType) {
|
||||
return `data:${normalizedMimeType};base64,${data}`;
|
||||
}
|
||||
|
||||
function parseMcpImageDetail(meta) {
|
||||
if (!isPlainObject(meta) || meta["codex/imageDetail"] !== "original") {
|
||||
return undefined;
|
||||
}
|
||||
return "original";
|
||||
}
|
||||
|
||||
function parseMcpToolResult(result) {
|
||||
if (typeof result === "string") {
|
||||
return { images: [], textCount: result.length > 0 ? 1 : 0 };
|
||||
@@ -1362,6 +1369,7 @@ function parseMcpToolResult(result) {
|
||||
if (item.type === "image") {
|
||||
images.push({
|
||||
image_url: normalizeMcpImageData(item.data, item.mimeType ?? item.mime_type),
|
||||
detail: parseMcpImageDetail(item._meta),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user