mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
Took over the work that @aaronl-openai started here: https://github.com/openai/codex/pull/10397 Now that app-server clients are able to set up custom tools (called `dynamic_tools` in app-server), we should expose a way for clients to pass in not just text, but also image outputs. This is something the Responses API already supports for function call outputs, where you can pass in either a string or an array of content outputs (text, image, file): https://platform.openai.com/docs/api-reference/responses/create#responses_create-input-input_item_list-item-function_tool_call_output-output-array-input_image So let's just plumb it through in Codex (with the caveat that we only support text and image for now). This is implemented end-to-end across app-server v2 protocol types and core tool handling. ## Breaking API change NOTE: This introduces a breaking change with dynamic tools, but I think it's ok since this concept was only recently introduced (https://github.com/openai/codex/pull/9539) and it's better to get the API contract correct. I don't think there are any real consumers of this yet (not even the Codex App). Old shape: `{ "output": "dynamic-ok", "success": true }` New shape: ``` { "contentItems": [ { "type": "inputText", "text": "dynamic-ok" }, { "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" } ] "success": true } ```
124 lines
4.3 KiB
Rust
124 lines
4.3 KiB
Rust
use crate::client_common::tools::ResponsesApiTool;
|
|
use crate::client_common::tools::ToolSpec;
|
|
use crate::codex::Session;
|
|
use crate::codex::TurnContext;
|
|
use crate::function_tool::FunctionCallError;
|
|
use crate::tools::context::ToolInvocation;
|
|
use crate::tools::context::ToolOutput;
|
|
use crate::tools::context::ToolPayload;
|
|
use crate::tools::registry::ToolHandler;
|
|
use crate::tools::registry::ToolKind;
|
|
use crate::tools::spec::JsonSchema;
|
|
use async_trait::async_trait;
|
|
use codex_protocol::config_types::ModeKind;
|
|
use codex_protocol::models::FunctionCallOutputBody;
|
|
use codex_protocol::plan_tool::UpdatePlanArgs;
|
|
use codex_protocol::protocol::EventMsg;
|
|
use std::collections::BTreeMap;
|
|
use std::sync::LazyLock;
|
|
|
|
pub struct PlanHandler;
|
|
|
|
pub static PLAN_TOOL: LazyLock<ToolSpec> = LazyLock::new(|| {
|
|
let mut plan_item_props = BTreeMap::new();
|
|
plan_item_props.insert("step".to_string(), JsonSchema::String { description: None });
|
|
plan_item_props.insert(
|
|
"status".to_string(),
|
|
JsonSchema::String {
|
|
description: Some("One of: pending, in_progress, completed".to_string()),
|
|
},
|
|
);
|
|
|
|
let plan_items_schema = JsonSchema::Array {
|
|
description: Some("The list of steps".to_string()),
|
|
items: Box::new(JsonSchema::Object {
|
|
properties: plan_item_props,
|
|
required: Some(vec!["step".to_string(), "status".to_string()]),
|
|
additional_properties: Some(false.into()),
|
|
}),
|
|
};
|
|
|
|
let mut properties = BTreeMap::new();
|
|
properties.insert(
|
|
"explanation".to_string(),
|
|
JsonSchema::String { description: None },
|
|
);
|
|
properties.insert("plan".to_string(), plan_items_schema);
|
|
|
|
ToolSpec::Function(ResponsesApiTool {
|
|
name: "update_plan".to_string(),
|
|
description: r#"Updates the task plan.
|
|
Provide an optional explanation and a list of plan items, each with a step and status.
|
|
At most one step can be in_progress at a time.
|
|
"#
|
|
.to_string(),
|
|
strict: false,
|
|
parameters: JsonSchema::Object {
|
|
properties,
|
|
required: Some(vec!["plan".to_string()]),
|
|
additional_properties: Some(false.into()),
|
|
},
|
|
})
|
|
});
|
|
|
|
#[async_trait]
|
|
impl ToolHandler for PlanHandler {
|
|
fn kind(&self) -> ToolKind {
|
|
ToolKind::Function
|
|
}
|
|
|
|
async fn handle(&self, invocation: ToolInvocation) -> Result<ToolOutput, FunctionCallError> {
|
|
let ToolInvocation {
|
|
session,
|
|
turn,
|
|
call_id,
|
|
payload,
|
|
..
|
|
} = invocation;
|
|
|
|
let arguments = match payload {
|
|
ToolPayload::Function { arguments } => arguments,
|
|
_ => {
|
|
return Err(FunctionCallError::RespondToModel(
|
|
"update_plan handler received unsupported payload".to_string(),
|
|
));
|
|
}
|
|
};
|
|
|
|
let content =
|
|
handle_update_plan(session.as_ref(), turn.as_ref(), arguments, call_id).await?;
|
|
|
|
Ok(ToolOutput::Function {
|
|
body: FunctionCallOutputBody::Text(content),
|
|
success: Some(true),
|
|
})
|
|
}
|
|
}
|
|
|
|
/// This function doesn't do anything useful. However, it gives the model a structured way to record its plan that clients can read and render.
|
|
/// So it's the _inputs_ to this function that are useful to clients, not the outputs and neither are actually useful for the model other
|
|
/// than forcing it to come up and document a plan (TBD how that affects performance).
|
|
pub(crate) async fn handle_update_plan(
|
|
session: &Session,
|
|
turn_context: &TurnContext,
|
|
arguments: String,
|
|
_call_id: String,
|
|
) -> Result<String, FunctionCallError> {
|
|
if turn_context.collaboration_mode.mode == ModeKind::Plan {
|
|
return Err(FunctionCallError::RespondToModel(
|
|
"update_plan is a TODO/checklist tool and is not allowed in Plan mode".to_string(),
|
|
));
|
|
}
|
|
let args = parse_update_plan_arguments(&arguments)?;
|
|
session
|
|
.send_event(turn_context, EventMsg::PlanUpdate(args))
|
|
.await;
|
|
Ok("Plan updated".to_string())
|
|
}
|
|
|
|
fn parse_update_plan_arguments(arguments: &str) -> Result<UpdatePlanArgs, FunctionCallError> {
|
|
serde_json::from_str::<UpdatePlanArgs>(arguments).map_err(|e| {
|
|
FunctionCallError::RespondToModel(format!("failed to parse function arguments: {e}"))
|
|
})
|
|
}
|