feat(app-server, core): allow text + image content items for dynamic tool outputs (#10567)

Took over the work that @aaronl-openai started here:
https://github.com/openai/codex/pull/10397

Now that app-server clients are able to set up custom tools (called
`dynamic_tools` in app-server), we should expose a way for clients to
pass in not just text, but also image outputs. This is something the
Responses API already supports for function call outputs, where you can
pass in either a string or an array of content outputs (text, image,
file):
https://platform.openai.com/docs/api-reference/responses/create#responses_create-input-input_item_list-item-function_tool_call_output-output-array-input_image

So let's just plumb it through in Codex (with the caveat that we only
support text and image for now). This is implemented end-to-end across
app-server v2 protocol types and core tool handling.

## Breaking API change
NOTE: This introduces a breaking change with dynamic tools, but I think
it's ok since this concept was only recently introduced
(https://github.com/openai/codex/pull/9539) and it's better to get the
API contract correct. I don't think there are any real consumers of this
yet (not even the Codex App).

Old shape:
`{ "output": "dynamic-ok", "success": true }`

New shape:
```
{
    "contentItems": [
      { "type": "inputText", "text": "dynamic-ok" },
      { "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" }
    ]
  "success": true
}
```

codex-rs/app-server/README.md

@@ -122,6 +122,7 @@ Start a fresh thread when you need a new Codex conversation.
     "approvalPolicy": "never",
     "sandbox": "workspaceWrite",
     "personality": "friendly",
+    // Experimental: requires opt-in
     "dynamicTools": [
         {
             "name": "lookup_ticket",
@@ -556,6 +557,41 @@ Order of messages:
 
 UI guidance for IDEs: surface an approval dialog as soon as the request arrives. The turn will proceed after the server receives a response to the approval request. The terminal `item/completed` notification will be sent with the appropriate status.
 
+### Dynamic tool calls (experimental)
+
+`dynamicTools` on `thread/start` and the corresponding `item/tool/call` request/response flow are experimental APIs. To enable them, set `initialize.params.capabilities.experimentalApi = true`.
+
+When a dynamic tool is invoked during a turn, the server sends an `item/tool/call` JSON-RPC request to the client:
+
+```json
+{
+  "method": "item/tool/call",
+  "id": 60,
+  "params": {
+    "threadId": "thr_123",
+    "turnId": "turn_123",
+    "callId": "call_123",
+    "tool": "lookup_ticket",
+    "arguments": { "id": "ABC-123" }
+  }
+}
+```
+
+The client must respond with content items. Use `inputText` for text and `inputImage` for image URLs/data URLs:
+
+```json
+{
+  "id": 60,
+  "result": {
+    "contentItems": [
+      { "type": "inputText", "text": "Ticket ABC-123 is open." },
+      { "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" }
+    ],
+    "success": true
+  }
+}
+```
+
 ## Skills
 
 Invoke a skill by including `$<skill-name>` in the text input. Add a `skill` input item (recommended) so the backend injects full skill instructions instead of relying on the model to resolve the name.

codex-rs/app-server/src/bespoke_event_handling.rs

@@ -88,6 +88,7 @@ use codex_core::protocol::TurnDiffEvent;
 use codex_core::review_format::format_review_findings_block;
 use codex_core::review_prompts;
 use codex_protocol::ThreadId;
+use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem as CoreDynamicToolCallOutputContentItem;
 use codex_protocol::dynamic_tools::DynamicToolResponse as CoreDynamicToolResponse;
 use codex_protocol::plan_tool::UpdatePlanArgs;
 use codex_protocol::protocol::ReviewOutputEvent;
@@ -351,8 +352,9 @@ pub(crate) async fn apply_bespoke_event_handling(
                     .submit(Op::DynamicToolResponse {
                         id: call_id.clone(),
                         response: CoreDynamicToolResponse {
-                            call_id,
-                            output: "dynamic tool calls require api v2".to_string(),
+                            content_items: vec![CoreDynamicToolCallOutputContentItem::InputText {
+                                text: "dynamic tool calls require api v2".to_string(),
+                            }],
                             success: false,
                         },
                     })

codex-rs/app-server/src/dynamic_tools.rs

@@ -1,5 +1,6 @@
 use codex_app_server_protocol::DynamicToolCallResponse;
 use codex_core::CodexThread;
+use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem as CoreDynamicToolCallOutputContentItem;
 use codex_protocol::dynamic_tools::DynamicToolResponse as CoreDynamicToolResponse;
 use codex_protocol::protocol::Op;
 use std::sync::Arc;
@@ -17,8 +18,9 @@ pub(crate) async fn on_call_response(
         Err(err) => {
             error!("request failed: {err:?}");
             let fallback = CoreDynamicToolResponse {
-                call_id: call_id.clone(),
-                output: "dynamic tool request failed".to_string(),
+                content_items: vec![CoreDynamicToolCallOutputContentItem::InputText {
+                    text: "dynamic tool request failed".to_string(),
+                }],
                 success: false,
             };
             if let Err(err) = conversation
@@ -37,14 +39,25 @@ pub(crate) async fn on_call_response(
     let response = serde_json::from_value::<DynamicToolCallResponse>(value).unwrap_or_else(|err| {
         error!("failed to deserialize DynamicToolCallResponse: {err}");
         DynamicToolCallResponse {
-            output: "dynamic tool response was invalid".to_string(),
+            content_items: vec![
+                codex_app_server_protocol::DynamicToolCallOutputContentItem::InputText {
+                    text: "dynamic tool response was invalid".to_string(),
+                },
+            ],
             success: false,
         }
     });
+
+    let DynamicToolCallResponse {
+        content_items,
+        success,
+    } = response;
     let response = CoreDynamicToolResponse {
-        call_id: call_id.clone(),
-        output: response.output,
-        success: response.success,
+        content_items: content_items
+            .into_iter()
+            .map(CoreDynamicToolCallOutputContentItem::from)
+            .collect(),
+        success,
     };
     if let Err(err) = conversation
         .submit(Op::DynamicToolResponse {

codex-rs/app-server/tests/suite/v2/dynamic_tools.rs

@@ -4,6 +4,7 @@ use app_test_support::McpProcess;
 use app_test_support::create_final_assistant_message_sse_response;
 use app_test_support::create_mock_responses_server_sequence_unchecked;
 use app_test_support::to_response;
+use codex_app_server_protocol::DynamicToolCallOutputContentItem;
 use codex_app_server_protocol::DynamicToolCallParams;
 use codex_app_server_protocol::DynamicToolCallResponse;
 use codex_app_server_protocol::DynamicToolSpec;
@@ -15,6 +16,9 @@ use codex_app_server_protocol::ThreadStartResponse;
 use codex_app_server_protocol::TurnStartParams;
 use codex_app_server_protocol::TurnStartResponse;
 use codex_app_server_protocol::UserInput as V2UserInput;
+use codex_protocol::models::FunctionCallOutputBody;
+use codex_protocol::models::FunctionCallOutputContentItem;
+use codex_protocol::models::FunctionCallOutputPayload;
 use core_test_support::responses;
 use pretty_assertions::assert_eq;
 use serde_json::Value;
@@ -111,7 +115,7 @@ async fn thread_start_injects_dynamic_tools_into_model_requests() -> Result<()>
 
 /// Exercises the full dynamic tool call path (server request, client response, model output).
 #[tokio::test]
-async fn dynamic_tool_call_round_trip_sends_output_to_model() -> Result<()> {
+async fn dynamic_tool_call_round_trip_sends_text_content_items_to_model() -> Result<()> {
     let call_id = "dyn-call-1";
     let tool_name = "demo_tool";
     let tool_args = json!({ "city": "Paris" });
@@ -200,7 +204,9 @@ async fn dynamic_tool_call_round_trip_sends_output_to_model() -> Result<()> {
 
     // Respond to the tool call so the model receives a function_call_output.
     let response = DynamicToolCallResponse {
-        output: "dynamic-ok".to_string(),
+        content_items: vec![DynamicToolCallOutputContentItem::InputText {
+            text: "dynamic-ok".to_string(),
+        }],
         success: true,
     };
     mcp.send_response(request_id, serde_json::to_value(response)?)
@@ -213,11 +219,171 @@ async fn dynamic_tool_call_round_trip_sends_output_to_model() -> Result<()> {
     .await??;
 
     let bodies = responses_bodies(&server).await?;
-    let output = bodies
+    let payload = bodies
         .iter()
-        .find_map(|body| function_call_output_text(body, call_id))
+        .find_map(|body| function_call_output_payload(body, call_id))
         .context("expected function_call_output in follow-up request")?;
-    assert_eq!(output, "dynamic-ok");
+    let expected_payload = FunctionCallOutputPayload::from_content_items(vec![
+        FunctionCallOutputContentItem::InputText {
+            text: "dynamic-ok".to_string(),
+        },
+    ]);
+    assert_eq!(payload, expected_payload);
+
+    Ok(())
+}
+
+/// Ensures dynamic tool call responses can include structured content items.
+#[tokio::test]
+async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<()> {
+    let call_id = "dyn-call-items-1";
+    let tool_name = "demo_tool";
+    let tool_args = json!({ "city": "Paris" });
+    let tool_call_arguments = serde_json::to_string(&tool_args)?;
+
+    let responses = vec![
+        responses::sse(vec![
+            responses::ev_response_created("resp-1"),
+            responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
+            responses::ev_completed("resp-1"),
+        ]),
+        create_final_assistant_message_sse_response("Done")?,
+    ];
+    let server = create_mock_responses_server_sequence_unchecked(responses).await;
+
+    let codex_home = TempDir::new()?;
+    create_config_toml(codex_home.path(), &server.uri())?;
+
+    let mut mcp = McpProcess::new(codex_home.path()).await?;
+    timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
+
+    let dynamic_tool = DynamicToolSpec {
+        name: tool_name.to_string(),
+        description: "Demo dynamic tool".to_string(),
+        input_schema: json!({
+            "type": "object",
+            "properties": {
+                "city": { "type": "string" }
+            },
+            "required": ["city"],
+            "additionalProperties": false,
+        }),
+    };
+
+    let thread_req = mcp
+        .send_thread_start_request(ThreadStartParams {
+            dynamic_tools: Some(vec![dynamic_tool]),
+            ..Default::default()
+        })
+        .await?;
+    let thread_resp: JSONRPCResponse = timeout(
+        DEFAULT_READ_TIMEOUT,
+        mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
+    )
+    .await??;
+    let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
+
+    let turn_req = mcp
+        .send_turn_start_request(TurnStartParams {
+            thread_id: thread.id.clone(),
+            input: vec![V2UserInput::Text {
+                text: "Run the tool".to_string(),
+                text_elements: Vec::new(),
+            }],
+            ..Default::default()
+        })
+        .await?;
+    let turn_resp: JSONRPCResponse = timeout(
+        DEFAULT_READ_TIMEOUT,
+        mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
+    )
+    .await??;
+    let TurnStartResponse { turn } = to_response::<TurnStartResponse>(turn_resp)?;
+
+    let request = timeout(
+        DEFAULT_READ_TIMEOUT,
+        mcp.read_stream_until_request_message(),
+    )
+    .await??;
+    let (request_id, params) = match request {
+        ServerRequest::DynamicToolCall { request_id, params } => (request_id, params),
+        other => panic!("expected DynamicToolCall request, got {other:?}"),
+    };
+
+    let expected = DynamicToolCallParams {
+        thread_id: thread.id,
+        turn_id: turn.id,
+        call_id: call_id.to_string(),
+        tool: tool_name.to_string(),
+        arguments: tool_args,
+    };
+    assert_eq!(params, expected);
+
+    let response_content_items = vec![
+        DynamicToolCallOutputContentItem::InputText {
+            text: "dynamic-ok".to_string(),
+        },
+        DynamicToolCallOutputContentItem::InputImage {
+            image_url: "data:image/png;base64,AAA".to_string(),
+        },
+    ];
+    let content_items = response_content_items
+        .clone()
+        .into_iter()
+        .map(|item| match item {
+            DynamicToolCallOutputContentItem::InputText { text } => {
+                FunctionCallOutputContentItem::InputText { text }
+            }
+            DynamicToolCallOutputContentItem::InputImage { image_url } => {
+                FunctionCallOutputContentItem::InputImage { image_url }
+            }
+        })
+        .collect::<Vec<FunctionCallOutputContentItem>>();
+    let response = DynamicToolCallResponse {
+        content_items: response_content_items,
+        success: true,
+    };
+    mcp.send_response(request_id, serde_json::to_value(response)?)
+        .await?;
+
+    timeout(
+        DEFAULT_READ_TIMEOUT,
+        mcp.read_stream_until_notification_message("turn/completed"),
+    )
+    .await??;
+
+    let bodies = responses_bodies(&server).await?;
+    let output_value = bodies
+        .iter()
+        .find_map(|body| function_call_output_raw_output(body, call_id))
+        .context("expected function_call_output output in follow-up request")?;
+    assert_eq!(
+        output_value,
+        json!([
+            {
+                "type": "input_text",
+                "text": "dynamic-ok"
+            },
+            {
+                "type": "input_image",
+                "image_url": "data:image/png;base64,AAA"
+            }
+        ])
+    );
+
+    let payload = bodies
+        .iter()
+        .find_map(|body| function_call_output_payload(body, call_id))
+        .context("expected function_call_output in follow-up request")?;
+    assert_eq!(
+        payload.body,
+        FunctionCallOutputBody::ContentItems(content_items.clone())
+    );
+    assert_eq!(payload.success, None);
+    assert_eq!(
+        serde_json::to_string(&payload)?,
+        serde_json::to_string(&content_items)?
+    );
 
     Ok(())
 }
@@ -248,7 +414,12 @@ fn find_tool<'a>(body: &'a Value, name: &str) -> Option<&'a Value> {
         })
 }
 
-fn function_call_output_text(body: &Value, call_id: &str) -> Option<String> {
+fn function_call_output_payload(body: &Value, call_id: &str) -> Option<FunctionCallOutputPayload> {
+    function_call_output_raw_output(body, call_id)
+        .and_then(|output| serde_json::from_value(output).ok())
+}
+
+fn function_call_output_raw_output(body: &Value, call_id: &str) -> Option<Value> {
     body.get("input")
         .and_then(Value::as_array)
         .and_then(|items| {
@@ -258,8 +429,7 @@ fn function_call_output_text(body: &Value, call_id: &str) -> Option<String> {
             })
         })
         .and_then(|item| item.get("output"))
-        .and_then(Value::as_str)
-        .map(str::to_string)
+        .cloned()
 }
 
 fn create_config_toml(codex_home: &Path, server_uri: &str) -> std::io::Result<()> {