Files
codex/codex-rs/core/src/tools/context.rs
T
Curtis 'Fjord' HawthorneandGitHub 125fbec317 Fix js_repl view_image attachments in nested tool calls (#12725)
## Summary

- Fix `js_repl` so `await codex.tool("view_image", { path })` actually
attaches the image to the active turn when called from inside the JS
REPL.
- Restore the behavior expected by the existing `js_repl`
image-attachment test.
- This is a follow-up to
[#12553](https://github.com/openai/codex/pull/12553), which changed
`view_image` to return structured image content.

## Root Cause

- [#12553](https://github.com/openai/codex/pull/12553) changed
`view_image` from directly injecting a pending user image message to
returning structured `function_call_output` content items.
- The nested tool-call bridge inside `js_repl` serialized that tool
response back to the JS runtime, but it did not mirror returned image
content into the active turn.
- As a result, `view_image` appeared to succeed inside `js_repl`, but no
`input_image` was actually attached for the outer turn.

## What Changed

- Updated the nested tool-call path in `js_repl` to inspect function
tool responses for structured content items.
- When a nested tool response includes `input_image` content, `js_repl`
now injects a corresponding user `Message` into the active turn before
returning the raw tool result back to the JS runtime.
- Kept the normal JSON result flow intact, so `codex.tool(...)` still
returns the original tool output object to JavaScript.

## Why

- `js_repl` documentation and tests already assume that `view_image` can
be used from inside the REPL to attach generated images to the model.
- Without this fix, the nested call path silently dropped that
attachment behavior.
2026-02-24 18:23:53 -08:00

290 lines
9.2 KiB
Rust

use crate::codex::Session;
use crate::codex::TurnContext;
use crate::tools::TELEMETRY_PREVIEW_MAX_BYTES;
use crate::tools::TELEMETRY_PREVIEW_MAX_LINES;
use crate::tools::TELEMETRY_PREVIEW_TRUNCATION_NOTICE;
use crate::turn_diff_tracker::TurnDiffTracker;
use codex_protocol::mcp::CallToolResult;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ShellToolCallParams;
use codex_utils_string::take_bytes_at_char_boundary;
use std::borrow::Cow;
use std::sync::Arc;
use tokio::sync::Mutex;
pub type SharedTurnDiffTracker = Arc<Mutex<TurnDiffTracker>>;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ToolCallSource {
Direct,
JsRepl,
}
#[derive(Clone)]
pub struct ToolInvocation {
pub session: Arc<Session>,
pub turn: Arc<TurnContext>,
pub tracker: SharedTurnDiffTracker,
pub call_id: String,
pub tool_name: String,
pub payload: ToolPayload,
}
#[derive(Clone, Debug)]
pub enum ToolPayload {
Function {
arguments: String,
},
Custom {
input: String,
},
LocalShell {
params: ShellToolCallParams,
},
Mcp {
server: String,
tool: String,
raw_arguments: String,
},
}
impl ToolPayload {
pub fn log_payload(&self) -> Cow<'_, str> {
match self {
ToolPayload::Function { arguments } => Cow::Borrowed(arguments),
ToolPayload::Custom { input } => Cow::Borrowed(input),
ToolPayload::LocalShell { params } => Cow::Owned(params.command.join(" ")),
ToolPayload::Mcp { raw_arguments, .. } => Cow::Borrowed(raw_arguments),
}
}
}
#[derive(Clone)]
pub enum ToolOutput {
Function {
// Canonical output body for function-style tools. This may be plain text
// or structured content items.
body: FunctionCallOutputBody,
success: Option<bool>,
},
Mcp {
result: Result<CallToolResult, String>,
},
}
impl ToolOutput {
pub fn log_preview(&self) -> String {
match self {
ToolOutput::Function { body, .. } => {
telemetry_preview(&body.to_text().unwrap_or_default())
}
ToolOutput::Mcp { result } => format!("{result:?}"),
}
}
pub fn success_for_logging(&self) -> bool {
match self {
ToolOutput::Function { success, .. } => success.unwrap_or(true),
ToolOutput::Mcp { result } => result.is_ok(),
}
}
pub fn into_response(self, call_id: &str, payload: &ToolPayload) -> ResponseInputItem {
match self {
ToolOutput::Function { body, success } => {
// `custom_tool_call` is the Responses API item type for freeform
// tools (`ToolSpec::Freeform`, e.g. freeform `apply_patch`).
// Those payloads must round-trip as `custom_tool_call_output`
// with plain string output.
if matches!(payload, ToolPayload::Custom { .. }) {
// Freeform/custom tools (`custom_tool_call`) use the custom
// output wire shape and remain string-only.
return ResponseInputItem::CustomToolCallOutput {
call_id: call_id.to_string(),
output: body.to_text().unwrap_or_default(),
};
}
// Function-style outputs (JSON function tools, including dynamic
// tools and MCP adaptation) preserve the exact body shape.
ResponseInputItem::FunctionCallOutput {
call_id: call_id.to_string(),
output: FunctionCallOutputPayload { body, success },
}
}
// Direct MCP response path for MCP tool result envelopes.
ToolOutput::Mcp { result } => ResponseInputItem::McpToolCallOutput {
call_id: call_id.to_string(),
result,
},
}
}
}
fn telemetry_preview(content: &str) -> String {
let truncated_slice = take_bytes_at_char_boundary(content, TELEMETRY_PREVIEW_MAX_BYTES);
let truncated_by_bytes = truncated_slice.len() < content.len();
let mut preview = String::new();
let mut lines_iter = truncated_slice.lines();
for idx in 0..TELEMETRY_PREVIEW_MAX_LINES {
match lines_iter.next() {
Some(line) => {
if idx > 0 {
preview.push('\n');
}
preview.push_str(line);
}
None => break,
}
}
let truncated_by_lines = lines_iter.next().is_some();
if !truncated_by_bytes && !truncated_by_lines {
return content.to_string();
}
if preview.len() < truncated_slice.len()
&& truncated_slice
.as_bytes()
.get(preview.len())
.is_some_and(|byte| *byte == b'\n')
{
preview.push('\n');
}
if !preview.is_empty() && !preview.ends_with('\n') {
preview.push('\n');
}
preview.push_str(TELEMETRY_PREVIEW_TRUNCATION_NOTICE);
preview
}
#[cfg(test)]
mod tests {
use super::*;
use codex_protocol::models::FunctionCallOutputContentItem;
use pretty_assertions::assert_eq;
#[test]
fn custom_tool_calls_should_roundtrip_as_custom_outputs() {
let payload = ToolPayload::Custom {
input: "patch".to_string(),
};
let response = ToolOutput::Function {
body: FunctionCallOutputBody::Text("patched".to_string()),
success: Some(true),
}
.into_response("call-42", &payload);
match response {
ResponseInputItem::CustomToolCallOutput { call_id, output } => {
assert_eq!(call_id, "call-42");
assert_eq!(output, "patched");
}
other => panic!("expected CustomToolCallOutput, got {other:?}"),
}
}
#[test]
fn function_payloads_remain_function_outputs() {
let payload = ToolPayload::Function {
arguments: "{}".to_string(),
};
let response = ToolOutput::Function {
body: FunctionCallOutputBody::Text("ok".to_string()),
success: Some(true),
}
.into_response("fn-1", &payload);
match response {
ResponseInputItem::FunctionCallOutput { call_id, output } => {
assert_eq!(call_id, "fn-1");
assert_eq!(output.text_content(), Some("ok"));
assert!(output.content_items().is_none());
assert_eq!(output.success, Some(true));
}
other => panic!("expected FunctionCallOutput, got {other:?}"),
}
}
#[test]
fn custom_tool_calls_can_derive_text_from_content_items() {
let payload = ToolPayload::Custom {
input: "patch".to_string(),
};
let response = ToolOutput::Function {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "line 1".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
},
FunctionCallOutputContentItem::InputText {
text: "line 2".to_string(),
},
]),
success: Some(true),
}
.into_response("call-99", &payload);
match response {
ResponseInputItem::CustomToolCallOutput { call_id, output } => {
assert_eq!(call_id, "call-99");
assert_eq!(output, "line 1\nline 2");
}
other => panic!("expected CustomToolCallOutput, got {other:?}"),
}
}
#[test]
fn log_preview_uses_content_items_when_plain_text_is_missing() {
let output = ToolOutput::Function {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "preview".to_string(),
},
]),
success: Some(true),
};
assert_eq!(output.log_preview(), "preview");
}
#[test]
fn telemetry_preview_returns_original_within_limits() {
let content = "short output";
assert_eq!(telemetry_preview(content), content);
}
#[test]
fn telemetry_preview_truncates_by_bytes() {
let content = "x".repeat(TELEMETRY_PREVIEW_MAX_BYTES + 8);
let preview = telemetry_preview(&content);
assert!(preview.contains(TELEMETRY_PREVIEW_TRUNCATION_NOTICE));
assert!(
preview.len()
<= TELEMETRY_PREVIEW_MAX_BYTES + TELEMETRY_PREVIEW_TRUNCATION_NOTICE.len() + 1
);
}
#[test]
fn telemetry_preview_truncates_by_lines() {
let content = (0..(TELEMETRY_PREVIEW_MAX_LINES + 5))
.map(|idx| format!("line {idx}"))
.collect::<Vec<_>>()
.join("\n");
let preview = telemetry_preview(&content);
let lines: Vec<&str> = preview.lines().collect();
assert!(lines.len() <= TELEMETRY_PREVIEW_MAX_LINES + 1);
assert_eq!(lines.last(), Some(&TELEMETRY_PREVIEW_TRUNCATION_NOTICE));
}
}