mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
Ahmed Ibrahim
f8937b7d86
## Why MCP authentication has distinct OAuth and ChatGPT-session flows. Representing that choice as `use_chatgpt_auth` makes one flow implicit and allows the configuration model to express the distinction only through a boolean. ChatGPT credential forwarding also needs a first-party trust boundary. A configurable `chatgpt_base_url` controls routing, but must not grant an MCP server permission to receive session credentials. This change builds on #29733, where the boolean was introduced. ## What changed - Replace `use_chatgpt_auth` with an `auth` field backed by the exhaustive `McpServerAuth` enum. - Support `auth = "oauth"` and `auth = "chatgpt"`, with OAuth remaining the default. - Trust only the origin derived from the existing hardcoded `CHATGPT_CODEX_BASE_URL` when granting ChatGPT auth to an MCP server. - Keep configured bearer tokens and authorization headers ahead of the selected authentication flow. - Update config writers, schema output, fixtures, and integration-test setup to use the enum. ## Verification Integration coverage exercises the complete streamable HTTP startup path in two independent configurations: - A directly constructed MCP configuration verifies that matching an overridden `chatgpt_base_url` does not grant ChatGPT auth. - A persisted `config.toml` containing an attacker-controlled `chatgpt_base_url` and `auth = "chatgpt"` verifies the same boundary through normal config parsing. Both tests complete MCP initialization and tool listing and assert that the full captured request sequence contains no authorization headers. Separate integration coverage verifies that configured authorization takes precedence over ChatGPT auth.
3908 lines
120 KiB
Rust
3908 lines
120 KiB
Rust
#![allow(clippy::unwrap_used)]
|
|
|
|
use anyhow::Result;
|
|
use base64::Engine;
|
|
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
|
use codex_config::types::McpServerConfig;
|
|
use codex_config::types::McpServerTransportConfig;
|
|
use codex_core::config::Config;
|
|
use codex_core::config::CurrentTimeReminderConfig;
|
|
use codex_extension_api::ExtensionRegistryBuilder;
|
|
use codex_features::CurrentTimeSource;
|
|
use codex_features::Feature;
|
|
use codex_login::CodexAuth;
|
|
use codex_models_manager::bundled_models_response;
|
|
use codex_protocol::config_types::WebSearchMode;
|
|
use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem;
|
|
use codex_protocol::dynamic_tools::DynamicToolFunctionSpec;
|
|
use codex_protocol::dynamic_tools::DynamicToolNamespaceSpec;
|
|
use codex_protocol::dynamic_tools::DynamicToolNamespaceTool;
|
|
use codex_protocol::dynamic_tools::DynamicToolResponse;
|
|
use codex_protocol::dynamic_tools::DynamicToolSpec;
|
|
use codex_protocol::models::PermissionProfile;
|
|
use codex_protocol::protocol::AskForApproval;
|
|
use codex_protocol::protocol::EventMsg;
|
|
use codex_protocol::protocol::Op;
|
|
use codex_protocol::user_input::UserInput;
|
|
use codex_web_search_extension::install as install_web_search_extension;
|
|
use core_test_support::apps_test_server::AppsTestServer;
|
|
use core_test_support::apps_test_server::AppsTestToolLoading;
|
|
use core_test_support::apps_test_server::DIRECT_CALENDAR_APP_ONLY_TOOL;
|
|
use core_test_support::apps_test_server::recorded_apps_tool_calls;
|
|
use core_test_support::apps_test_server::search_capable_apps_builder;
|
|
use core_test_support::assert_regex_match;
|
|
use core_test_support::responses;
|
|
use core_test_support::responses::ResponseMock;
|
|
use core_test_support::responses::ResponsesRequest;
|
|
use core_test_support::responses::ev_assistant_message;
|
|
use core_test_support::responses::ev_completed;
|
|
use core_test_support::responses::ev_custom_tool_call;
|
|
use core_test_support::responses::ev_response_created;
|
|
use core_test_support::responses::sse;
|
|
use core_test_support::skip_if_no_network;
|
|
use core_test_support::skip_if_wine_exec;
|
|
use core_test_support::stdio_server_bin;
|
|
use core_test_support::test_codex::TestCodex;
|
|
use core_test_support::test_codex::test_codex;
|
|
use core_test_support::test_codex::turn_permission_fields;
|
|
use core_test_support::wait_for_event;
|
|
use core_test_support::wait_for_event_match;
|
|
use core_test_support::wait_for_mcp_server;
|
|
use image::DynamicImage;
|
|
use image::GenericImageView;
|
|
use image::ImageBuffer;
|
|
use image::Rgba;
|
|
use pretty_assertions::assert_eq;
|
|
use serde_json::Value;
|
|
use std::collections::HashMap;
|
|
use std::collections::HashSet;
|
|
use std::fs;
|
|
use std::io::Cursor;
|
|
use std::path::Path;
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
use std::time::Instant;
|
|
use wiremock::Mock;
|
|
use wiremock::MockServer;
|
|
use wiremock::ResponseTemplate;
|
|
use wiremock::matchers::method;
|
|
use wiremock::matchers::path;
|
|
|
|
fn custom_tool_output_items(req: &ResponsesRequest, call_id: &str) -> Vec<Value> {
|
|
match req.custom_tool_call_output(call_id).get("output") {
|
|
Some(Value::Array(items)) => items.clone(),
|
|
Some(Value::String(text)) => {
|
|
vec![serde_json::json!({ "type": "input_text", "text": text })]
|
|
}
|
|
_ => panic!("custom tool output should be serialized as text or content items"),
|
|
}
|
|
}
|
|
|
|
fn tool_names(body: &Value) -> Vec<String> {
|
|
body.get("tools")
|
|
.and_then(Value::as_array)
|
|
.map(|tools| {
|
|
tools
|
|
.iter()
|
|
.filter_map(|tool| {
|
|
tool.get("name")
|
|
.or_else(|| tool.get("type"))
|
|
.and_then(Value::as_str)
|
|
.map(str::to_string)
|
|
})
|
|
.collect()
|
|
})
|
|
.unwrap_or_default()
|
|
}
|
|
|
|
fn function_tool_output_items(req: &ResponsesRequest, call_id: &str) -> Vec<Value> {
|
|
match req.function_call_output(call_id).get("output") {
|
|
Some(Value::Array(items)) => items.clone(),
|
|
Some(Value::String(text)) => {
|
|
vec![serde_json::json!({ "type": "input_text", "text": text })]
|
|
}
|
|
_ => panic!("function tool output should be serialized as text or content items"),
|
|
}
|
|
}
|
|
|
|
fn text_item(items: &[Value], index: usize) -> &str {
|
|
items[index]
|
|
.get("text")
|
|
.and_then(Value::as_str)
|
|
.expect("content item should be input_text")
|
|
}
|
|
|
|
fn extract_running_cell_id(text: &str) -> String {
|
|
text.strip_prefix("Script running with cell ID ")
|
|
.and_then(|rest| rest.split('\n').next())
|
|
.expect("running header should contain a cell ID")
|
|
.to_string()
|
|
}
|
|
|
|
fn wait_for_file_source(path: &Path) -> Result<String> {
|
|
let quoted_path = shlex::try_join([path.to_string_lossy().as_ref()])?;
|
|
let command = format!("if [ -f {quoted_path} ]; then printf ready; fi");
|
|
Ok(format!(
|
|
r#"while ((await tools.exec_command({{ cmd: {command:?} }})).output !== "ready") {{
|
|
}}"#
|
|
))
|
|
}
|
|
|
|
fn custom_tool_output_body_and_success(
|
|
req: &ResponsesRequest,
|
|
call_id: &str,
|
|
) -> (String, Option<bool>) {
|
|
let (content, success) = req
|
|
.custom_tool_call_output_content_and_success(call_id)
|
|
.expect("custom tool output should be present");
|
|
let items = custom_tool_output_items(req, call_id);
|
|
let text_items = items
|
|
.iter()
|
|
.filter_map(|item| item.get("text").and_then(Value::as_str))
|
|
.collect::<Vec<_>>();
|
|
let output = match text_items.as_slice() {
|
|
[] => content.unwrap_or_default(),
|
|
[only] => (*only).to_string(),
|
|
[_, rest @ ..] => rest.concat(),
|
|
};
|
|
(output, success)
|
|
}
|
|
|
|
fn custom_tool_output_last_non_empty_text(req: &ResponsesRequest, call_id: &str) -> Option<String> {
|
|
match req.custom_tool_call_output(call_id).get("output") {
|
|
Some(Value::String(text)) if !text.trim().is_empty() => Some(text.clone()),
|
|
Some(Value::Array(items)) => items
|
|
.iter()
|
|
.filter_map(|item| item.get("text").and_then(Value::as_str))
|
|
.rfind(|text| !text.trim().is_empty())
|
|
.map(str::to_string),
|
|
Some(Value::String(_))
|
|
| Some(Value::Object(_))
|
|
| Some(Value::Number(_))
|
|
| Some(Value::Bool(_))
|
|
| Some(Value::Null)
|
|
| None => None,
|
|
}
|
|
}
|
|
|
|
async fn run_code_mode_turn(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
run_code_mode_turn_with_config(server, prompt, code, |_| {}).await
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_config(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
configure: impl FnOnce(&mut Config) + Send + 'static,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
run_code_mode_turn_with_model_and_config(server, prompt, code, "test-gpt-5.1-codex", configure)
|
|
.await
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_model_and_config(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
model: &'static str,
|
|
configure: impl FnOnce(&mut Config) + Send + 'static,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
let mut builder = test_codex().with_model(model).with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
configure(config);
|
|
});
|
|
let test = builder.build(server).await?;
|
|
|
|
responses::mount_sse_once(
|
|
server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let second_mock = responses::mount_sse_once(
|
|
server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn(prompt).await?;
|
|
Ok((test, second_mock))
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_call_standalone_web_search() -> Result<()> {
|
|
assert_code_mode_standalone_web_search(WebSearchMode::Live, serde_json::json!(true)).await
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_call_indexed_standalone_web_search() -> Result<()> {
|
|
assert_code_mode_standalone_web_search(WebSearchMode::Indexed, serde_json::json!("indexed"))
|
|
.await
|
|
}
|
|
|
|
async fn assert_code_mode_standalone_web_search(
|
|
web_search_mode: WebSearchMode,
|
|
expected_external_web_access: Value,
|
|
) -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
Mock::given(method("POST"))
|
|
.and(path("/v1/alpha/search"))
|
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
|
"output": "Search result",
|
|
})))
|
|
.expect(1)
|
|
.mount(&server)
|
|
.await;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
const result = await tools.web__run({
|
|
search_query: [{ q: "standalone web search" }],
|
|
});
|
|
text(result);
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let follow_up_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let auth = CodexAuth::from_api_key("dummy");
|
|
let auth_manager = codex_core::test_support::auth_manager_from_auth(auth.clone());
|
|
let mut extension_builder = ExtensionRegistryBuilder::<Config>::new();
|
|
install_web_search_extension(&mut extension_builder, auth_manager);
|
|
let mut builder = test_codex()
|
|
.with_auth(auth)
|
|
.with_extensions(Arc::new(extension_builder.build()))
|
|
.with_model("test-gpt-5.1-codex")
|
|
.with_config(move |config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("code mode should be enabled");
|
|
config
|
|
.features
|
|
.enable(Feature::StandaloneWebSearch)
|
|
.expect("standalone web search should be enabled");
|
|
config
|
|
.web_search_mode
|
|
.set(web_search_mode)
|
|
.expect("web search mode should be accepted");
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
test.submit_turn("Search the web from code mode").await?;
|
|
|
|
let search_request = server
|
|
.received_requests()
|
|
.await
|
|
.expect("received requests should be available")
|
|
.into_iter()
|
|
.find(|request| request.url.path() == "/v1/alpha/search")
|
|
.expect("standalone search request should be sent");
|
|
let search_body = search_request
|
|
.body_json::<Value>()
|
|
.expect("search request body should be JSON");
|
|
assert_eq!(
|
|
search_body["model"],
|
|
serde_json::json!("test-gpt-5.1-codex")
|
|
);
|
|
assert_eq!(
|
|
search_body["commands"],
|
|
serde_json::json!({
|
|
"search_query": [{"q": "standalone web search"}],
|
|
})
|
|
);
|
|
assert_eq!(
|
|
search_body["settings"],
|
|
serde_json::json!({
|
|
"allowed_callers": ["direct"],
|
|
"external_web_access": expected_external_web_access,
|
|
})
|
|
);
|
|
assert_eq!(
|
|
custom_tool_output_last_non_empty_text(&follow_up_mock.single_request(), "call-1"),
|
|
Some("Search result".to_string())
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_rmcp(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
run_code_mode_turn_with_rmcp_model(server, prompt, code, "test-gpt-5.1-codex").await
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_rmcp_model(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
model: &'static str,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
run_code_mode_turn_with_rmcp_config(
|
|
server, prompt, code, model, /*code_mode_only*/ false,
|
|
/*non_prefixed_mcp_tool_names*/ false,
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_rmcp_mode(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
code_mode_only: bool,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
run_code_mode_turn_with_rmcp_config(
|
|
server,
|
|
prompt,
|
|
code,
|
|
"test-gpt-5.1-codex",
|
|
code_mode_only,
|
|
/*non_prefixed_mcp_tool_names*/ false,
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn run_code_mode_turn_with_rmcp_config(
|
|
server: &MockServer,
|
|
prompt: &str,
|
|
code: &str,
|
|
model: &'static str,
|
|
code_mode_only: bool,
|
|
non_prefixed_mcp_tool_names: bool,
|
|
) -> Result<(TestCodex, ResponseMock)> {
|
|
let rmcp_test_server_bin = stdio_server_bin()?;
|
|
let mut builder = test_codex().with_model(model).with_config(move |config| {
|
|
let _ = if code_mode_only {
|
|
config.features.enable(Feature::CodeModeOnly)
|
|
} else {
|
|
config.features.enable(Feature::CodeMode)
|
|
};
|
|
if non_prefixed_mcp_tool_names {
|
|
let _ = config.features.enable(Feature::NonPrefixedMcpToolNames);
|
|
}
|
|
|
|
let mut servers = config.mcp_servers.get().clone();
|
|
servers.insert(
|
|
"rmcp".to_string(),
|
|
McpServerConfig {
|
|
auth: Default::default(),
|
|
transport: McpServerTransportConfig::Stdio {
|
|
command: rmcp_test_server_bin,
|
|
args: Vec::new(),
|
|
env: Some(HashMap::from([(
|
|
"MCP_TEST_VALUE".to_string(),
|
|
"propagated-env".to_string(),
|
|
)])),
|
|
env_vars: Vec::new(),
|
|
cwd: None,
|
|
},
|
|
environment_id: "local".to_string(),
|
|
enabled: true,
|
|
required: false,
|
|
supports_parallel_tool_calls: false,
|
|
disabled_reason: None,
|
|
startup_timeout_sec: Some(Duration::from_secs(10)),
|
|
tool_timeout_sec: None,
|
|
default_tools_approval_mode: None,
|
|
enabled_tools: None,
|
|
disabled_tools: None,
|
|
scopes: None,
|
|
oauth: None,
|
|
oauth_resource: None,
|
|
tools: HashMap::new(),
|
|
},
|
|
);
|
|
config
|
|
.mcp_servers
|
|
.set(servers)
|
|
.expect("test mcp servers should accept any configuration");
|
|
});
|
|
let test = builder.build(server).await?;
|
|
wait_for_mcp_server(&test.codex, "rmcp").await?;
|
|
|
|
responses::mount_sse_once(
|
|
server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let second_mock = responses::mount_sse_once(
|
|
server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn(prompt).await?;
|
|
Ok((test, second_mock))
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_return_exec_command_output() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to run exec_command",
|
|
r#"
|
|
text(JSON.stringify(await tools.exec_command({ cmd: "printf code_mode_exec_marker" })));
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
let parsed: Value = serde_json::from_str(text_item(&items, /*index*/ 1))?;
|
|
assert!(
|
|
parsed
|
|
.get("chunk_id")
|
|
.and_then(Value::as_str)
|
|
.is_some_and(|chunk_id| !chunk_id.is_empty())
|
|
);
|
|
assert_eq!(
|
|
parsed.get("output").and_then(Value::as_str),
|
|
Some("code_mode_exec_marker"),
|
|
);
|
|
assert_eq!(parsed.get("exit_code").and_then(Value::as_i64), Some(0));
|
|
assert!(parsed.get("wall_time_seconds").is_some());
|
|
assert!(parsed.get("session_id").is_none());
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_only_restricts_prompt_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let resp_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let mut builder = test_codex().with_config(|config| {
|
|
let _ = config.features.enable(Feature::CodeModeOnly);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
test.submit_turn("list tools in code mode only").await?;
|
|
|
|
let first_body = resp_mock.single_request().body_json();
|
|
assert_eq!(
|
|
tool_names(&first_body),
|
|
vec![
|
|
"exec".to_string(),
|
|
"wait".to_string(),
|
|
"request_user_input".to_string(),
|
|
"web_search".to_string()
|
|
]
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_only_guides_all_tools_search_and_calls_deferred_app_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let apps_server = AppsTestServer::mount_searchable(&server).await?;
|
|
let resp_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
const tool = ALL_TOOLS.find(
|
|
({ name }) => name === "mcp__codex_apps__calendar_timezone_option_99"
|
|
);
|
|
if (!tool) {
|
|
text(JSON.stringify({ found: false }));
|
|
} else {
|
|
const result = await tools[tool.name]({ timezone: "UTC" });
|
|
text(JSON.stringify({
|
|
found: true,
|
|
isError: Boolean(result.isError),
|
|
text: result.content?.[0]?.text ?? "",
|
|
}));
|
|
}
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let follow_up_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let apps_base_url = apps_server.chatgpt_base_url.clone();
|
|
let mut builder = test_codex()
|
|
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
|
|
.with_config(move |config| {
|
|
config
|
|
.features
|
|
.enable(Feature::Apps)
|
|
.expect("test config should allow feature update");
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("test config should allow feature update");
|
|
config
|
|
.features
|
|
.enable(Feature::CodeModeOnly)
|
|
.expect("test config should allow feature update");
|
|
let mut model_catalog =
|
|
bundled_models_response().expect("bundled models.json should parse");
|
|
let model = model_catalog
|
|
.models
|
|
.iter_mut()
|
|
.find(|model| model.slug == "gpt-5.4")
|
|
.expect("gpt-5.4 exists in bundled models.json");
|
|
config.chatgpt_base_url = apps_base_url;
|
|
config.model = Some("gpt-5.4".to_string());
|
|
model.supports_search_tool = true;
|
|
config.model_catalog = Some(model_catalog);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
test.submit_turn("inspect tools in code mode only").await?;
|
|
|
|
let first_body = resp_mock.single_request().body_json();
|
|
assert_eq!(
|
|
tool_names(&first_body),
|
|
vec![
|
|
"exec".to_string(),
|
|
"wait".to_string(),
|
|
"request_user_input".to_string(),
|
|
"web_search".to_string(),
|
|
"image_generation".to_string()
|
|
]
|
|
);
|
|
|
|
let exec_description = first_body
|
|
.get("tools")
|
|
.and_then(Value::as_array)
|
|
.and_then(|tools| {
|
|
tools.iter().find_map(|tool| {
|
|
if tool
|
|
.get("name")
|
|
.or_else(|| tool.get("type"))
|
|
.and_then(Value::as_str)
|
|
== Some("exec")
|
|
{
|
|
tool.get("description").and_then(Value::as_str)
|
|
} else {
|
|
None
|
|
}
|
|
})
|
|
})
|
|
.expect("exec description should be present");
|
|
assert!(exec_description.contains("filter `ALL_TOOLS` by `name` and `description`"));
|
|
assert!(!exec_description.contains("calendar_timezone_option_99"));
|
|
|
|
let request = follow_up_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code_mode_only deferred app tool call failed unexpectedly: {output}"
|
|
);
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(
|
|
parsed,
|
|
serde_json::json!({
|
|
"found": true,
|
|
"isError": false,
|
|
"text": "called calendar_timezone_option_99 for at with ",
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn app_only_tools_are_not_visible_or_runnable_by_code_mode_model() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let apps_server =
|
|
AppsTestServer::mount_with_app_only_tool(&server, AppsTestToolLoading::Searchable).await?;
|
|
let code = format!(
|
|
r#"
|
|
const visibleTool = ALL_TOOLS.find(({{ name }}) => name === {visible_tool_name:?});
|
|
const tool = ALL_TOOLS.find(({{ name }}) => name === {tool_name:?});
|
|
let error = null;
|
|
try {{
|
|
await tools[{tool_name:?}]({{}});
|
|
}} catch (caught) {{
|
|
error = String(caught);
|
|
}}
|
|
text(JSON.stringify({{
|
|
visibleListed: visibleTool !== undefined,
|
|
listed: tool !== undefined,
|
|
callable: typeof tools[{tool_name:?}] === "function",
|
|
error,
|
|
}}));
|
|
"#,
|
|
visible_tool_name = "mcp__codex_apps__calendar_timezone_option_99",
|
|
tool_name = DIRECT_CALENDAR_APP_ONLY_TOOL,
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let mut builder = search_capable_apps_builder(apps_server.chatgpt_base_url.clone())
|
|
.with_config(|config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CodeMode)
|
|
.expect("test config should allow feature update");
|
|
config
|
|
.features
|
|
.enable(Feature::CodeModeOnly)
|
|
.expect("test config should allow feature update");
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
test.submit_turn("try to call the app-only calendar tool through exec")
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code mode visibility check should complete successfully: {output}"
|
|
);
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(parsed["visibleListed"], true);
|
|
assert_eq!(parsed["listed"], false);
|
|
assert_eq!(parsed["callable"], false);
|
|
assert!(
|
|
parsed["error"]
|
|
.as_str()
|
|
.is_some_and(|error| error.contains("is not a function")),
|
|
"app-only code mode call should fail before MCP dispatch: {parsed:?}"
|
|
);
|
|
assert!(
|
|
recorded_apps_tool_calls(&server).await.is_empty(),
|
|
"app-only code mode call should not reach the MCP server"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_only_can_call_nested_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
const output = await tools.exec_command({ cmd: "printf code_mode_only_nested_tool_marker" });
|
|
text(output.output);
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let follow_up_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let mut builder = test_codex().with_config(|config| {
|
|
let _ = config.features.enable(Feature::CodeModeOnly);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
test.submit_turn("use exec to run nested tool in code mode only")
|
|
.await?;
|
|
|
|
let request = follow_up_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code_mode_only nested tool call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(output, "code_mode_only_nested_tool_marker");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_update_plan_nested_tool_result_is_empty_object() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to run update_plan",
|
|
r#"
|
|
const result = await tools.update_plan({
|
|
plan: [{ step: "Run update_plan from code mode", status: "in_progress" }],
|
|
});
|
|
text(JSON.stringify(result));
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec update_plan call failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(parsed, serde_json::json!({}));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_get_context_remaining_returns_structured_result() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_config(
|
|
&server,
|
|
"use exec to get remaining context",
|
|
r#"
|
|
const result = await tools.get_context_remaining({});
|
|
text(JSON.stringify(result));
|
|
"#,
|
|
|config| {
|
|
config.model_context_window = Some(10_000);
|
|
config
|
|
.features
|
|
.enable(Feature::TokenBudget)
|
|
.expect("test config should allow token budget");
|
|
},
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec get_context_remaining call failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(
|
|
parsed,
|
|
serde_json::json!({
|
|
"tokens_left": 9000,
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_current_time_returns_structured_result() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_config(
|
|
&server,
|
|
"use exec to get the current time",
|
|
r#"
|
|
const result = await tools.clock__curr_time({});
|
|
text(JSON.stringify(result));
|
|
"#,
|
|
|config| {
|
|
config
|
|
.features
|
|
.enable(Feature::CurrentTimeReminder)
|
|
.expect("test config should allow current-time reminders");
|
|
config.current_time_reminder = Some(CurrentTimeReminderConfig {
|
|
reminder_interval_seconds: 3_000,
|
|
clock_source: CurrentTimeSource::System,
|
|
..CurrentTimeReminderConfig::default()
|
|
});
|
|
},
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec clock.curr_time call failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
let current_time = parsed
|
|
.get("current_time")
|
|
.and_then(Value::as_str)
|
|
.expect("clock.curr_time should return current_time");
|
|
assert_regex_match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} UTC$", current_time);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "flaky on windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_nested_tool_calls_can_run_in_parallel() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex()
|
|
.with_model("test-gpt-5.1-codex")
|
|
.with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
let warmup_code = r#"
|
|
const args = {
|
|
sleep_after_ms: 10,
|
|
barrier: {
|
|
id: "code-mode-parallel-tools-warmup",
|
|
participants: 2,
|
|
timeout_ms: 1_000,
|
|
},
|
|
};
|
|
|
|
await Promise.all([
|
|
tools.test_sync_tool(args),
|
|
tools.test_sync_tool(args),
|
|
]);
|
|
"#;
|
|
let code = r#"
|
|
const args = {
|
|
sleep_after_ms: 300,
|
|
barrier: {
|
|
id: "code-mode-parallel-tools",
|
|
participants: 2,
|
|
timeout_ms: 1_000,
|
|
},
|
|
};
|
|
|
|
const results = await Promise.all([
|
|
tools.test_sync_tool(args),
|
|
tools.test_sync_tool(args),
|
|
]);
|
|
|
|
text(JSON.stringify(results));
|
|
"#;
|
|
|
|
let response_mock = responses::mount_sse_sequence(
|
|
&server,
|
|
vec![
|
|
sse(vec![
|
|
ev_response_created("resp-warm-1"),
|
|
ev_custom_tool_call("call-warm-1", "exec", warmup_code),
|
|
ev_completed("resp-warm-1"),
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("msg-warm-1", "warmup done"),
|
|
ev_completed("resp-warm-2"),
|
|
]),
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
],
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("warm up nested tools in parallel").await?;
|
|
|
|
let start = Instant::now();
|
|
test.submit_turn("run nested tools in parallel").await?;
|
|
let duration = start.elapsed();
|
|
|
|
assert!(
|
|
duration < Duration::from_millis(1_600),
|
|
"expected nested tools to finish in parallel, got {duration:?}",
|
|
);
|
|
|
|
let req = response_mock
|
|
.last_request()
|
|
.expect("parallel code mode run should send a completion request");
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
assert_eq!(items.len(), 2);
|
|
assert_eq!(text_item(&items, /*index*/ 1), "[\"ok\",\"ok\"]");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
// This model uses token-based tool-output truncation, giving the downstream
|
|
// history assertions a stable `…N tokens truncated…` marker.
|
|
const TOKEN_POLICY_TEST_MODEL: &str = "gpt-5.4";
|
|
|
|
// A nested `exec_command` limit applies to `result.output` inside JavaScript.
|
|
// The outer code-mode and history budgets apply after the script calls `text`.
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_nested_limit_formats_truncated_result_with_warning() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"
|
|
const result = await tools.exec_command({
|
|
cmd: "printf '0123456789012345678901234567890123456789'",
|
|
max_output_tokens: 5
|
|
});
|
|
text(result.output);
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
assert_eq!(
|
|
text_item(
|
|
&custom_tool_output_items(&second_mock.single_request(), "call-1"),
|
|
/*index*/ 1
|
|
),
|
|
"Warning: truncated output (original token count: 10)\nTotal output lines: 1\n\n0123456789…5 tokens truncated…0123456789"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_nested_limit_preserves_result_variable_before_default_history_truncation()
|
|
-> Result<()> {
|
|
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"only part of nested exec_command stdout reaches the code-mode result"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"// @exec: {"max_output_tokens": 20000}
|
|
const result = await tools.exec_command({
|
|
cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\"",
|
|
max_output_tokens: 20000
|
|
});
|
|
const resultVariableWasTruncated = result.output.length !== 50000;
|
|
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
|
|
"#,
|
|
TOKEN_POLICY_TEST_MODEL,
|
|
|_| {},
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
let output = text_item(&items, /*index*/ 1);
|
|
assert_regex_match(
|
|
r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$",
|
|
output,
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_nested_limit_truncates_result_variable_when_exceeded() -> Result<()> {
|
|
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"only part of nested exec_command stdout reaches the code-mode result"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"// @exec: {"max_output_tokens": 25000}
|
|
const result = await tools.exec_command({
|
|
cmd: "python3 -c \"import sys; sys.stdout.write('A' * 90000)\"",
|
|
max_output_tokens: 20000
|
|
});
|
|
const resultVariableWasTruncated = result.output.includes("…2500 tokens truncated…");
|
|
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
|
|
"#,
|
|
TOKEN_POLICY_TEST_MODEL,
|
|
|_| {},
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
let output = text_item(&items, /*index*/ 1);
|
|
// The nested 20,000-token budget leaves about 80,000 characters. This
|
|
// ceiling independently proves that history applied its smaller cap.
|
|
assert!(
|
|
output.len() < 60_000,
|
|
"expected history to truncate the emitted value, got {} bytes",
|
|
output.len()
|
|
);
|
|
// The boolean describes the nested result; the marker below comes from
|
|
// history truncating the value emitted with `text` afterward.
|
|
assert_regex_match(
|
|
r"(?s)^Variable truncated: True\. Variable: .*…\d+ tokens truncated…A+$",
|
|
output,
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_nested_limit_preserves_result_variable_before_configured_history_truncation()
|
|
-> Result<()> {
|
|
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"only part of nested exec_command stdout reaches the code-mode result"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"// @exec: {"max_output_tokens": 20000}
|
|
const result = await tools.exec_command({
|
|
cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\"",
|
|
max_output_tokens: 20000
|
|
});
|
|
const resultVariableWasTruncated = result.output.length !== 50000;
|
|
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
|
|
"#,
|
|
TOKEN_POLICY_TEST_MODEL,
|
|
|config| {
|
|
config.tool_output_token_limit = Some(50);
|
|
},
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
let output = text_item(&items, /*index*/ 1);
|
|
// The 50-token override must shrink this 50,000-character value far below
|
|
// what the default 10,000-token history cap would retain.
|
|
assert!(
|
|
output.len() < 1_000,
|
|
"expected configured history cap to truncate the emitted value, got {} bytes",
|
|
output.len()
|
|
);
|
|
assert_regex_match(
|
|
r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$",
|
|
output,
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_without_nested_limit_preserves_result_variable_before_default_history_truncation()
|
|
-> Result<()> {
|
|
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"only part of nested exec_command stdout reaches the code-mode result"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"// @exec: {"max_output_tokens": 20000}
|
|
const result = await tools.exec_command({
|
|
cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\""
|
|
});
|
|
const resultVariableWasTruncated = result.output.length !== 50000;
|
|
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
|
|
"#,
|
|
TOKEN_POLICY_TEST_MODEL,
|
|
|_| {},
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
let output = text_item(&items, /*index*/ 1);
|
|
assert_regex_match(
|
|
r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$",
|
|
output,
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_without_nested_limit_preserves_result_variable_before_configured_history_truncation()
|
|
-> Result<()> {
|
|
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
|
|
skip_if_wine_exec!(
|
|
Ok(()),
|
|
"only part of nested exec_command stdout reaches the code-mode result"
|
|
);
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"// @exec: {"max_output_tokens": 20000}
|
|
const result = await tools.exec_command({
|
|
cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\""
|
|
});
|
|
const resultVariableWasTruncated = result.output.length !== 50000;
|
|
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
|
|
"#,
|
|
TOKEN_POLICY_TEST_MODEL,
|
|
|config| {
|
|
config.tool_output_token_limit = Some(50);
|
|
},
|
|
)
|
|
.await?;
|
|
|
|
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
|
|
let output = text_item(&items, /*index*/ 1);
|
|
// The 50-token override must shrink this 50,000-character value far below
|
|
// what the default 10,000-token history cap would retain.
|
|
assert!(
|
|
output.len() < 1_000,
|
|
"expected configured history cap to truncate the emitted value, got {} bytes",
|
|
output.len()
|
|
);
|
|
assert_regex_match(
|
|
r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$",
|
|
output,
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
// The outer directive limits output after JavaScript emits it; it does not
|
|
// limit `result.output` returned by the nested command.
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_outer_limit_truncates_emitted_output() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec_command from code mode",
|
|
r#"// @exec: {"max_output_tokens": 5}
|
|
const result = await tools.exec_command({
|
|
cmd: "printf '0123456789012345678901234567890123456789'"
|
|
});
|
|
text(result.output);
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
assert_eq!(
|
|
text_item(
|
|
&custom_tool_output_items(&second_mock.single_request(), "call-1"),
|
|
/*index*/ 1
|
|
),
|
|
"Warning: truncated output (original token count: 10)\nTotal output lines: 1\n\n0123456789…5 tokens truncated…0123456789"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_returns_accumulated_output_when_script_fails() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use code_mode to surface script failures",
|
|
r#"
|
|
text("before crash");
|
|
text("still before crash");
|
|
throw new Error("boom");
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
assert_eq!(items.len(), 4);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script failed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&items, /*index*/ 1), "before crash");
|
|
assert_eq!(text_item(&items, /*index*/ 2), "still before crash");
|
|
assert_regex_match(
|
|
r#"(?sx)
|
|
\A
|
|
Script\ error:\n
|
|
Error:\ boom\n
|
|
(?:\s+at\ .+\n?)+
|
|
\z
|
|
"#,
|
|
text_item(&items, /*index*/ 3),
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exec_surfaces_handler_errors_as_exceptions() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"surface nested tool handler failures as script exceptions",
|
|
r#"
|
|
try {
|
|
await tools.exec_command({});
|
|
text("no-exception");
|
|
} catch (error) {
|
|
text(`caught:${error?.message ?? String(error)}`);
|
|
}
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let request = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"script should catch the nested tool error: {output}"
|
|
);
|
|
assert!(
|
|
output.contains("caught:"),
|
|
"expected caught exception text in output: {output}"
|
|
);
|
|
assert!(
|
|
!output.contains("no-exception"),
|
|
"nested tool error should not allow success path: {output}"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_yield_and_resume_with_wait() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let phase_2_gate = test.workspace_path("code-mode-phase-2.ready");
|
|
let phase_3_gate = test.workspace_path("code-mode-phase-3.ready");
|
|
let phase_2_wait = wait_for_file_source(&phase_2_gate)?;
|
|
let phase_3_wait = wait_for_file_source(&phase_3_gate)?;
|
|
|
|
let code = format!(
|
|
r#"
|
|
text("phase 1");
|
|
yield_control();
|
|
{phase_2_wait}
|
|
text("phase 2");
|
|
{phase_3_wait}
|
|
text("phase 3");
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start the long exec").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&first_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&first_items, /*index*/ 1), "phase 1");
|
|
let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
responses::ev_function_call(
|
|
"call-2",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": cell_id.clone(),
|
|
"yield_time_ms": 1_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "still waiting"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
fs::write(&phase_2_gate, "ready")?;
|
|
test.submit_turn("wait again").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
let second_items = function_tool_output_items(&second_request, "call-2");
|
|
assert_eq!(second_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&second_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(
|
|
extract_running_cell_id(text_item(&second_items, /*index*/ 0)),
|
|
cell_id
|
|
);
|
|
assert_eq!(text_item(&second_items, /*index*/ 1), "phase 2");
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-5"),
|
|
responses::ev_function_call(
|
|
"call-3",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": cell_id.clone(),
|
|
"yield_time_ms": 1_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-5"),
|
|
]),
|
|
)
|
|
.await;
|
|
let third_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-3", "done"),
|
|
ev_completed("resp-6"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
fs::write(&phase_3_gate, "ready")?;
|
|
test.submit_turn("wait for completion").await?;
|
|
|
|
let third_request = third_completion.single_request();
|
|
let third_items = function_tool_output_items(&third_request, "call-3");
|
|
assert_eq!(third_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&third_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&third_items, /*index*/ 1), "phase 3");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_yield_and_termination_are_not_starved_by_runtime_output() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
// Exact controller arbitration is covered by deterministic code-mode contract tests. Keep
|
|
// this end-to-end load bounded while exercising a substantial runtime output backlog.
|
|
let code = r#"// @exec: {"yield_time_ms": 0, "max_output_tokens": 16}
|
|
for (let index = 0; index < 16_384; index++) {
|
|
text(`event ${index}`);
|
|
}
|
|
while (true) {}
|
|
"#;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
tokio::time::timeout(
|
|
Duration::from_secs(5),
|
|
test.submit_turn("start the busy loop"),
|
|
)
|
|
.await??;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 1);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&first_items, /*index*/ 0),
|
|
);
|
|
let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
responses::ev_function_call(
|
|
"call-2",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": cell_id.clone(),
|
|
"terminate": true,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "terminated"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("terminate it").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
let second_items = function_tool_output_items(&second_request, "call-2");
|
|
assert!(!second_items.is_empty());
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script terminated\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&second_items, /*index*/ 0),
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_run_multiple_yielded_sessions() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let session_a_gate = test.workspace_path("code-mode-session-a.ready");
|
|
let session_b_gate = test.workspace_path("code-mode-session-b.ready");
|
|
let session_a_wait = wait_for_file_source(&session_a_gate)?;
|
|
let session_b_wait = wait_for_file_source(&session_b_gate)?;
|
|
|
|
let session_a_code = format!(
|
|
r#"
|
|
text("session a start");
|
|
yield_control();
|
|
{session_a_wait}
|
|
text("session a done");
|
|
"#
|
|
);
|
|
let session_b_code = format!(
|
|
r#"
|
|
text("session b start");
|
|
yield_control();
|
|
{session_b_wait}
|
|
text("session b done");
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &session_a_code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "session a waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start session a").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 2);
|
|
let session_a_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
assert_eq!(text_item(&first_items, /*index*/ 1), "session a start");
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
ev_custom_tool_call("call-2", "exec", &session_b_code),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "session b waiting"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start session b").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
let second_items = custom_tool_output_items(&second_request, "call-2");
|
|
assert_eq!(second_items.len(), 2);
|
|
let session_b_id = extract_running_cell_id(text_item(&second_items, /*index*/ 0));
|
|
assert_eq!(text_item(&second_items, /*index*/ 1), "session b start");
|
|
assert_ne!(session_a_id, session_b_id);
|
|
|
|
fs::write(&session_a_gate, "ready")?;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-5"),
|
|
responses::ev_function_call(
|
|
"call-3",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": session_a_id.clone(),
|
|
"yield_time_ms": 1_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-5"),
|
|
]),
|
|
)
|
|
.await;
|
|
let third_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-3", "session a done"),
|
|
ev_completed("resp-6"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("wait session a").await?;
|
|
|
|
let third_request = third_completion.single_request();
|
|
let third_items = function_tool_output_items(&third_request, "call-3");
|
|
assert_eq!(third_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&third_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&third_items, /*index*/ 1), "session a done");
|
|
|
|
fs::write(&session_b_gate, "ready")?;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-7"),
|
|
responses::ev_function_call(
|
|
"call-4",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": session_b_id.clone(),
|
|
"yield_time_ms": 1_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-7"),
|
|
]),
|
|
)
|
|
.await;
|
|
let fourth_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-4", "session b done"),
|
|
ev_completed("resp-8"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("wait session b").await?;
|
|
|
|
let fourth_request = fourth_completion.single_request();
|
|
let fourth_items = function_tool_output_items(&fourth_request, "call-4");
|
|
assert_eq!(fourth_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&fourth_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&fourth_items, /*index*/ 1), "session b done");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_concurrent_cells_merge_only_the_stored_values_they_write() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let first_gate = test.workspace_path("code-mode-first-store.ready");
|
|
let first_wait = wait_for_file_source(&first_gate)?;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-init",
|
|
"exec",
|
|
r#"
|
|
store("a", 1);
|
|
store("b", 2);
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "initialized"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("initialize stored values").await?;
|
|
|
|
let first_code = format!(
|
|
r#"
|
|
store("a", 3);
|
|
yield_control();
|
|
{first_wait}
|
|
"#
|
|
);
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
ev_custom_tool_call("call-first", "exec", &first_code),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_started = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "first pending"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start first store").await?;
|
|
|
|
let first_request = first_started.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-first");
|
|
let first_cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-5"),
|
|
ev_custom_tool_call("call-second", "exec", r#"store("b", 4);"#),
|
|
ev_completed("resp-5"),
|
|
]),
|
|
)
|
|
.await;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-3", "second complete"),
|
|
ev_completed("resp-6"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("write the second key").await?;
|
|
|
|
fs::write(&first_gate, "ready")?;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-7"),
|
|
responses::ev_function_call(
|
|
"call-wait",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": first_cell_id,
|
|
"yield_time_ms": 1_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-7"),
|
|
]),
|
|
)
|
|
.await;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-4", "first completed"),
|
|
ev_completed("resp-8"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("complete the first store").await?;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-9"),
|
|
ev_custom_tool_call(
|
|
"call-check",
|
|
"exec",
|
|
r#"text(JSON.stringify({ a: load("a"), b: load("b") }));"#,
|
|
),
|
|
ev_completed("resp-9"),
|
|
]),
|
|
)
|
|
.await;
|
|
let check_response = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-5", "checked"),
|
|
ev_completed("resp-10"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("check merged stored values").await?;
|
|
|
|
let check_request = check_response.single_request();
|
|
let stored_values: Value = serde_json::from_str(
|
|
&custom_tool_output_last_non_empty_text(&check_request, "call-check")
|
|
.expect("checking stored values should emit JSON"),
|
|
)?;
|
|
assert_eq!(stored_values, serde_json::json!({ "a": 3, "b": 4 }));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_wait_can_terminate_and_continue() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let termination_gate = test.workspace_path("code-mode-terminate.ready");
|
|
let termination_wait = wait_for_file_source(&termination_gate)?;
|
|
|
|
let code = format!(
|
|
r#"
|
|
text("phase 1");
|
|
yield_control();
|
|
{termination_wait}
|
|
text("phase 2");
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start the long exec").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 2);
|
|
let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
assert_eq!(text_item(&first_items, /*index*/ 1), "phase 1");
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
responses::ev_function_call(
|
|
"call-2",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": cell_id.clone(),
|
|
"terminate": true,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "terminated"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("terminate it").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
let second_items = function_tool_output_items(&second_request, "call-2");
|
|
assert_eq!(second_items.len(), 1);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script terminated\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&second_items, /*index*/ 0),
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-5"),
|
|
ev_custom_tool_call(
|
|
"call-3",
|
|
"exec",
|
|
r#"
|
|
text("after terminate");
|
|
"#,
|
|
),
|
|
ev_completed("resp-5"),
|
|
]),
|
|
)
|
|
.await;
|
|
let third_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-3", "done"),
|
|
ev_completed("resp-6"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("run another exec").await?;
|
|
|
|
let third_request = third_completion.single_request();
|
|
let third_items = custom_tool_output_items(&third_request, "call-3");
|
|
assert_eq!(third_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&third_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&third_items, /*index*/ 1), "after terminate");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_wait_returns_error_for_unknown_session() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
responses::ev_function_call(
|
|
"call-1",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": "999999",
|
|
"yield_time_ms": 1_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("wait on an unknown exec cell").await?;
|
|
|
|
let request = completion.single_request();
|
|
let (_, success) = request
|
|
.function_call_output_content_and_success("call-1")
|
|
.expect("function tool output should be present");
|
|
assert_ne!(success, Some(true));
|
|
|
|
let items = function_tool_output_items(&request, "call-1");
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script failed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
assert_eq!(
|
|
text_item(&items, /*index*/ 1),
|
|
"Script error:\nexec cell 999999 not found"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_wait_terminate_returns_completed_session_if_it_finished_after_yield_control()
|
|
-> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let session_a_gate = test.workspace_path("code-mode-session-a-finished.ready");
|
|
let session_b_gate = test.workspace_path("code-mode-session-b-blocked.ready");
|
|
let session_a_done_marker = test.workspace_path("code-mode-session-a-done.txt");
|
|
let session_a_wait = wait_for_file_source(&session_a_gate)?;
|
|
let session_b_wait = wait_for_file_source(&session_b_gate)?;
|
|
let session_a_done_marker_quoted =
|
|
shlex::try_join([session_a_done_marker.to_string_lossy().as_ref()])?;
|
|
let session_a_done_command = format!("printf done > {session_a_done_marker_quoted}");
|
|
|
|
let session_a_code = format!(
|
|
r#"
|
|
text("session a start");
|
|
yield_control();
|
|
{session_a_wait}
|
|
text("session a done");
|
|
await tools.exec_command({{ cmd: {session_a_done_command:?} }});
|
|
"#
|
|
);
|
|
let session_b_code = format!(
|
|
r#"
|
|
text("session b start");
|
|
yield_control();
|
|
{session_b_wait}
|
|
text("session b done");
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &session_a_code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "session a waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start session a").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 2);
|
|
let session_a_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
assert_eq!(text_item(&first_items, /*index*/ 1), "session a start");
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
ev_custom_tool_call("call-2", "exec", &session_b_code),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "session b waiting"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start session b").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
let second_items = custom_tool_output_items(&second_request, "call-2");
|
|
assert_eq!(second_items.len(), 2);
|
|
let session_b_id = extract_running_cell_id(text_item(&second_items, /*index*/ 0));
|
|
assert_eq!(text_item(&second_items, /*index*/ 1), "session b start");
|
|
|
|
fs::write(&session_a_gate, "ready")?;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-5"),
|
|
responses::ev_function_call(
|
|
"call-3",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": session_b_id.clone(),
|
|
"yield_time_ms": 1_000,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-5"),
|
|
]),
|
|
)
|
|
.await;
|
|
let third_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-3", "session b still waiting"),
|
|
ev_completed("resp-6"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("wait session b").await?;
|
|
|
|
let third_request = third_completion.single_request();
|
|
let third_items = function_tool_output_items(&third_request, "call-3");
|
|
assert_eq!(third_items.len(), 1);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&third_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(
|
|
extract_running_cell_id(text_item(&third_items, /*index*/ 0)),
|
|
session_b_id
|
|
);
|
|
|
|
for _ in 0..100 {
|
|
if session_a_done_marker.exists() {
|
|
break;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
}
|
|
assert!(session_a_done_marker.exists());
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-7"),
|
|
responses::ev_function_call(
|
|
"call-4",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": session_a_id.clone(),
|
|
"terminate": true,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-7"),
|
|
]),
|
|
)
|
|
.await;
|
|
let fourth_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-4", "session a already done"),
|
|
ev_completed("resp-8"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("terminate session a").await?;
|
|
|
|
let fourth_request = fourth_completion.single_request();
|
|
let fourth_items = function_tool_output_items(&fourth_request, "call-4");
|
|
match fourth_items.len() {
|
|
1 => {
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script terminated\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&fourth_items, /*index*/ 0),
|
|
);
|
|
}
|
|
2 => {
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script (?:completed|terminated)\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&fourth_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&fourth_items, /*index*/ 1), "session a done");
|
|
}
|
|
other => panic!("unexpected number of content items: {other}"),
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_background_keeps_running_on_later_turn_without_wait() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let resumed_file = test.workspace_path("code-mode-yield-resumed.txt");
|
|
let resumed_file_quoted = shlex::try_join([resumed_file.to_string_lossy().as_ref()])?;
|
|
let write_file_command = format!("printf resumed > {resumed_file_quoted}");
|
|
let wait_for_file_command =
|
|
format!("while [ ! -f {resumed_file_quoted} ]; do sleep 0.01; done; printf ready");
|
|
let code = format!(
|
|
r#"
|
|
text("before yield");
|
|
yield_control();
|
|
await tools.exec_command({{ cmd: {write_file_command:?} }});
|
|
text("after yield");
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "exec yielded"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start yielded exec").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script running with cell ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&first_items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&first_items, /*index*/ 1), "before yield");
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
responses::ev_function_call(
|
|
"call-2",
|
|
"exec_command",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cmd": wait_for_file_command,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "file appeared"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("wait for resumed file").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
assert!(
|
|
second_request
|
|
.function_call_output_text("call-2")
|
|
.is_some_and(|output| output.ends_with("ready"))
|
|
);
|
|
assert_eq!(fs::read_to_string(&resumed_file)?, "resumed");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_wait_uses_its_own_max_tokens_budget() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
let completion_gate = test.workspace_path("code-mode-max-tokens.ready");
|
|
let completion_wait = wait_for_file_source(&completion_gate)?;
|
|
|
|
let code = format!(
|
|
r#"// @exec: {{"max_output_tokens": 100}}
|
|
text("phase 1");
|
|
yield_control();
|
|
{completion_wait}
|
|
text("token one token two token three token four token five token six token seven");
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "waiting"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("start the long exec").await?;
|
|
|
|
let first_request = first_completion.single_request();
|
|
let first_items = custom_tool_output_items(&first_request, "call-1");
|
|
assert_eq!(first_items.len(), 2);
|
|
assert_eq!(text_item(&first_items, /*index*/ 1), "phase 1");
|
|
let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
|
|
|
|
fs::write(&completion_gate, "ready")?;
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
responses::ev_function_call(
|
|
"call-2",
|
|
"wait",
|
|
&serde_json::to_string(&serde_json::json!({
|
|
"cell_id": cell_id.clone(),
|
|
"yield_time_ms": 1_000,
|
|
"max_tokens": 6,
|
|
}))?,
|
|
),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_completion = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "done"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("wait for completion").await?;
|
|
|
|
let second_request = second_completion.single_request();
|
|
let second_items = function_tool_output_items(&second_request, "call-2");
|
|
assert_eq!(second_items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&second_items, /*index*/ 0),
|
|
);
|
|
let expected_pattern = r#"(?sx)
|
|
\A
|
|
Warning:\ truncated\ output\ \(original\ token\ count:\ \d+\)\n
|
|
Total\ output\ lines:\ 1\n
|
|
\n
|
|
.*…\d+\ tokens\ truncated….*
|
|
\z
|
|
"#;
|
|
assert_regex_match(expected_pattern, text_item(&second_items, /*index*/ 1));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_output_serialized_text_via_global_helper() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to return structured text",
|
|
r#"
|
|
text({ json: true });
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
eprintln!(
|
|
"hidden dynamic tool raw output: {}",
|
|
req.custom_tool_call_output("call-1")
|
|
);
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(output, r#"{"json":true}"#);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_resume_after_set_timeout() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to wait for a timeout",
|
|
r#"
|
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
text("timer done");
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec setTimeout call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(output, "timer done");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_notify_injects_additional_exec_tool_output_into_active_context() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec notify helper",
|
|
r#"
|
|
notify("code_mode_notify_marker");
|
|
await tools.test_sync_tool({});
|
|
text("done");
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let has_notify_output = req
|
|
.inputs_of_type("custom_tool_call_output")
|
|
.iter()
|
|
.any(|item| {
|
|
item.get("call_id").and_then(serde_json::Value::as_str) == Some("call-1")
|
|
&& item
|
|
.get("output")
|
|
.and_then(serde_json::Value::as_str)
|
|
.is_some_and(|text| text.contains("code_mode_notify_marker"))
|
|
&& item.get("name").and_then(serde_json::Value::as_str) == Some("exec")
|
|
});
|
|
assert!(
|
|
has_notify_output,
|
|
"expected notify marker in custom_tool_call_output item: {:?}",
|
|
req.input()
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exit_stops_script_immediately() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to stop script early with exit helper",
|
|
r#"
|
|
text("before");
|
|
exit();
|
|
text("after");
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec exit helper call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&items, /*index*/ 1), "before");
|
|
assert_eq!(output, "before");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_surfaces_text_stringify_errors() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to return circular text",
|
|
r#"
|
|
const circular = {};
|
|
circular.self = circular;
|
|
text(circular);
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = req
|
|
.custom_tool_call_output_content_and_success("call-1")
|
|
.expect("custom tool output should be present");
|
|
assert_ne!(
|
|
success,
|
|
Some(true),
|
|
"circular stringify unexpectedly succeeded"
|
|
);
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script failed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
assert!(text_item(&items, /*index*/ 1).contains("Script error:"));
|
|
assert!(text_item(&items, /*index*/ 1).contains("Converting circular structure to JSON"));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_output_images_via_global_helper() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to return images",
|
|
r#"
|
|
image("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==");
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code_mode image output failed unexpectedly"
|
|
);
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
assert_eq!(
|
|
items[1],
|
|
serde_json::json!({
|
|
"type": "input_image",
|
|
"image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==",
|
|
"detail": "high"
|
|
}),
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_replaces_malformed_image() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to return an image",
|
|
r#"image("data:image/png;base64,AAA");"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(success, Some(false));
|
|
assert_eq!(items.len(), 2);
|
|
assert_eq!(
|
|
items[1],
|
|
serde_json::json!({
|
|
"type": "input_text",
|
|
"text": "image content omitted because it could not be processed"
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_resizes_explicit_original_image() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let original_dimensions = (6401, 100);
|
|
let image = ImageBuffer::from_pixel(
|
|
original_dimensions.0,
|
|
original_dimensions.1,
|
|
Rgba([20, 40, 60, 255]),
|
|
);
|
|
let mut encoded = Cursor::new(Vec::new());
|
|
DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?;
|
|
let image_data_url = format!(
|
|
"data:image/png;base64,{}",
|
|
BASE64_STANDARD.encode(encoded.into_inner())
|
|
);
|
|
let code = format!(
|
|
"image({}, \"original\");",
|
|
serde_json::to_string(&image_data_url)?
|
|
);
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
|
|
&server,
|
|
"use exec to return a large original-detail image",
|
|
&code,
|
|
"gpt-5.3-codex",
|
|
|_| {},
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(success, Some(false));
|
|
let resized_url = items[1]["image_url"]
|
|
.as_str()
|
|
.expect("code mode image output should contain a data URL");
|
|
assert_eq!(items[1]["detail"], "original");
|
|
let (_, resized_base64) = resized_url
|
|
.split_once(',')
|
|
.expect("resized image should contain a data URL prefix");
|
|
let resized_bytes = BASE64_STANDARD.decode(resized_base64)?;
|
|
let resized = image::load_from_memory(&resized_bytes)?;
|
|
let resized_dimensions = resized.dimensions();
|
|
assert_eq!(resized_dimensions, (6000, 94));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_image_helper_rejects_remote_url() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"use exec to return a remote image",
|
|
r#"image("https://example.com/image.jpg");"#,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(true),
|
|
"code_mode remote image URL unexpectedly succeeded"
|
|
);
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script failed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
assert_eq!(
|
|
text_item(&items, /*index*/ 1),
|
|
concat!(
|
|
"Script error:\n",
|
|
"Tool call failed: remote image URLs are not supported in tool outputs. ",
|
|
"Pass a base64 data URI instead"
|
|
)
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex()
|
|
.with_model("gpt-5.3-codex")
|
|
.with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
let image_bytes = BASE64_STANDARD.decode(
|
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==",
|
|
)?;
|
|
let image_path = test.cwd_path().join("code_mode_view_image.png");
|
|
fs::write(&image_path, image_bytes)?;
|
|
|
|
let image_path_json = serde_json::to_string(&image_path.to_string_lossy().to_string())?;
|
|
let code = format!(
|
|
r#"
|
|
const out = await tools.view_image({{ path: {image_path_json}, detail: "original" }});
|
|
image(out);
|
|
"#
|
|
);
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", &code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("use exec to call view_image and emit its image output")
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code_mode view_image call failed unexpectedly"
|
|
);
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
|
|
assert_eq!(
|
|
items[1].get("type").and_then(Value::as_str),
|
|
Some("input_image")
|
|
);
|
|
|
|
let emitted_image_url = items[1]
|
|
.get("image_url")
|
|
.and_then(Value::as_str)
|
|
.expect("image helper should emit an input_image item with image_url");
|
|
assert!(emitted_image_url.starts_with("data:image/png;base64,"));
|
|
assert_eq!(
|
|
items[1].get("detail").and_then(Value::as_str),
|
|
Some("original")
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_use_mcp_image_result_with_image_helper() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const out = await tools.mcp__rmcp__image_scenario({
|
|
scenario: "image_only_original_detail",
|
|
});
|
|
const imageItem = out.content.find((item) => item.type === "image");
|
|
image(imageItem);
|
|
"#;
|
|
|
|
let (_test, second_mock) = run_code_mode_turn_with_rmcp_model(
|
|
&server,
|
|
"use exec to call the rmcp image scenario tool and emit its image output",
|
|
code,
|
|
"gpt-5.3-codex",
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code_mode mcp image scenario call failed unexpectedly"
|
|
);
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
|
|
assert_eq!(
|
|
items[1].get("type").and_then(Value::as_str),
|
|
Some("input_image")
|
|
);
|
|
|
|
let emitted_image_url = items[1]
|
|
.get("image_url")
|
|
.and_then(Value::as_str)
|
|
.expect("image helper should emit an input_image item with image_url");
|
|
assert!(emitted_image_url.starts_with("data:image/png;base64,"));
|
|
assert_eq!(
|
|
items[1].get("detail").and_then(Value::as_str),
|
|
Some("original")
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_apply_patch_via_nested_tool() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let file_name = "code_mode_apply_patch.txt";
|
|
let patch = format!(
|
|
"*** Begin Patch\n*** Add File: {file_name}\n+hello from code_mode\n*** End Patch\n"
|
|
);
|
|
let code = format!("text(await tools.apply_patch({patch:?}));\n");
|
|
|
|
let (test, second_mock) =
|
|
run_code_mode_turn(&server, "use exec to run apply_patch", &code).await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let items = custom_tool_output_items(&req, "call-1");
|
|
let (_, success) = req
|
|
.custom_tool_call_output_content_and_success("call-1")
|
|
.expect("custom tool output should be present");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec apply_patch call failed unexpectedly: {items:?}"
|
|
);
|
|
assert_eq!(items.len(), 2);
|
|
assert_regex_match(
|
|
concat!(
|
|
r"(?s)\A",
|
|
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
|
|
),
|
|
text_item(&items, /*index*/ 0),
|
|
);
|
|
assert_eq!(text_item(&items, /*index*/ 1), "{}");
|
|
|
|
let file_path = test.cwd_path().join(file_name);
|
|
assert_eq!(fs::read_to_string(&file_path)?, "hello from code_mode\n");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_print_structured_mcp_tool_result_fields() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const { content, structuredContent, isError } = await tools.mcp__rmcp__echo({
|
|
message: "ping",
|
|
});
|
|
text(
|
|
`echo=${structuredContent?.echo ?? "missing"}\n` +
|
|
`env=${structuredContent?.env ?? "missing"}\n` +
|
|
`isError=${String(isError)}\n` +
|
|
`contentLength=${content.length}`
|
|
);
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn_with_rmcp(&server, "use exec to run the rmcp echo tool", code).await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec rmcp echo call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(
|
|
output,
|
|
"echo=ECHOING: ping
|
|
env=propagated-env
|
|
isError=false
|
|
contentLength=0"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_only_can_call_mcp_tool() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const result = await tools.mcp__rmcp__echo({ message: "ping" });
|
|
text(`echo=${result.structuredContent?.echo ?? "missing"}`);
|
|
"#;
|
|
|
|
let (_test, second_mock) = run_code_mode_turn_with_rmcp_mode(
|
|
&server,
|
|
"use exec to run the rmcp echo tool in code mode only",
|
|
code,
|
|
/*code_mode_only*/ true,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"code_mode_only rmcp tool call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(output, "echo=ECHOING: ping");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exposes_mcp_tools_on_global_tools_object() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const { content, structuredContent, isError } = await tools.mcp__rmcp__echo({
|
|
message: "ping",
|
|
});
|
|
text(
|
|
`hasEcho=${String(Object.keys(tools).includes("mcp__rmcp__echo"))}\n` +
|
|
`echoType=${typeof tools.mcp__rmcp__echo}\n` +
|
|
`echo=${structuredContent?.echo ?? "missing"}\n` +
|
|
`isError=${String(isError)}\n` +
|
|
`contentLength=${content.length}`
|
|
);
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn_with_rmcp(&server, "use exec to inspect the global tools object", code)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec global rmcp access failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(
|
|
output,
|
|
"hasEcho=true
|
|
echoType=function
|
|
echo=ECHOING: ping
|
|
isError=false
|
|
contentLength=0"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_uses_non_prefixed_mcp_tool_names_when_feature_enabled() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const result = await tools.rmcp__echo({ message: "ping" });
|
|
text(JSON.stringify({
|
|
hasNonPrefixedEcho: typeof tools.rmcp__echo === "function",
|
|
hasPrefixedEcho: typeof tools.mcp__rmcp__echo === "function",
|
|
echo: result.structuredContent?.echo ?? "missing",
|
|
}));
|
|
"#;
|
|
|
|
let (_test, second_mock) = run_code_mode_turn_with_rmcp_config(
|
|
&server,
|
|
"use exec to inspect non-prefixed MCP names",
|
|
code,
|
|
"test-gpt-5.1-codex",
|
|
/*code_mode_only*/ false,
|
|
/*non_prefixed_mcp_tool_names*/ true,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec non-prefixed rmcp access failed unexpectedly: {output}"
|
|
);
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(
|
|
parsed,
|
|
serde_json::json!({
|
|
"hasNonPrefixedEcho": true,
|
|
"hasPrefixedEcho": false,
|
|
"echo": "ECHOING: ping",
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exposes_namespaced_mcp_tools_on_global_tools_object() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
text(JSON.stringify({
|
|
hasExecCommand: typeof tools.exec_command === "function",
|
|
hasNamespacedEcho: typeof tools.mcp__rmcp__echo === "function",
|
|
}));
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn_with_rmcp(&server, "use exec to inspect the global tools object", code)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec global tools inspection failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(&output)?;
|
|
assert_eq!(
|
|
parsed,
|
|
serde_json::json!({
|
|
"hasExecCommand": !cfg!(windows),
|
|
"hasNamespacedEcho": true,
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exposes_normalized_illegal_mcp_tool_names() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const result = await tools.mcp__rmcp__echo_tool({ message: "ping" });
|
|
text(`echo=${result.structuredContent.echo}`);
|
|
"#;
|
|
|
|
let (_test, second_mock) = run_code_mode_turn_with_rmcp(
|
|
&server,
|
|
"use exec to call a normalized rmcp tool name",
|
|
code,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec normalized rmcp tool call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(output, "echo=ECHOING: ping");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_lists_global_scope_items() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
text(JSON.stringify(Object.getOwnPropertyNames(globalThis).sort()));
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn_with_rmcp(&server, "use exec to inspect global scope", code).await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec global scope inspection failed unexpectedly: {output}"
|
|
);
|
|
let globals = serde_json::from_str::<Vec<String>>(&output)?;
|
|
let globals = globals.into_iter().collect::<HashSet<_>>();
|
|
let expected = [
|
|
"AggregateError",
|
|
"ALL_TOOLS",
|
|
"Array",
|
|
"ArrayBuffer",
|
|
"AsyncDisposableStack",
|
|
"BigInt",
|
|
"BigInt64Array",
|
|
"BigUint64Array",
|
|
"Boolean",
|
|
"clearTimeout",
|
|
"DataView",
|
|
"Date",
|
|
"DisposableStack",
|
|
"Error",
|
|
"EvalError",
|
|
"FinalizationRegistry",
|
|
"Float16Array",
|
|
"Float32Array",
|
|
"Float64Array",
|
|
"Function",
|
|
"Infinity",
|
|
"Int16Array",
|
|
"Int32Array",
|
|
"Int8Array",
|
|
"Intl",
|
|
"Iterator",
|
|
"JSON",
|
|
"Map",
|
|
"Math",
|
|
"NaN",
|
|
"Number",
|
|
"Object",
|
|
"Promise",
|
|
"Proxy",
|
|
"RangeError",
|
|
"ReferenceError",
|
|
"Reflect",
|
|
"RegExp",
|
|
"Set",
|
|
"String",
|
|
"SuppressedError",
|
|
"Symbol",
|
|
"SyntaxError",
|
|
"Temporal",
|
|
"TypeError",
|
|
"URIError",
|
|
"Uint16Array",
|
|
"Uint32Array",
|
|
"Uint8Array",
|
|
"Uint8ClampedArray",
|
|
"WeakMap",
|
|
"WeakRef",
|
|
"WeakSet",
|
|
"__codexContentItems",
|
|
"add_content",
|
|
"decodeURI",
|
|
"decodeURIComponent",
|
|
"encodeURI",
|
|
"encodeURIComponent",
|
|
"escape",
|
|
"exit",
|
|
"eval",
|
|
"generatedImage",
|
|
"globalThis",
|
|
"image",
|
|
"isFinite",
|
|
"isNaN",
|
|
"load",
|
|
"notify",
|
|
"parseFloat",
|
|
"parseInt",
|
|
"setTimeout",
|
|
"store",
|
|
"text",
|
|
"tools",
|
|
"undefined",
|
|
"unescape",
|
|
"yield_control",
|
|
];
|
|
for g in &globals {
|
|
assert!(
|
|
expected.contains(&g.as_str()),
|
|
"unexpected global {g} in {globals:?}"
|
|
);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exports_all_tools_metadata_for_builtin_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const tool = ALL_TOOLS.find(({ name }) => name === "view_image");
|
|
text(JSON.stringify(tool));
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn(&server, "use exec to inspect ALL_TOOLS", code).await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec ALL_TOOLS lookup failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(
|
|
&custom_tool_output_last_non_empty_text(&req, "call-1")
|
|
.expect("exec ALL_TOOLS lookup should emit JSON"),
|
|
)?;
|
|
assert_eq!(
|
|
parsed,
|
|
serde_json::json!({
|
|
"name": "view_image",
|
|
"description": "View a local image file from the filesystem when visual inspection is needed. Use this for images already available on disk.\n\nexec tool declaration:\n```ts\ndeclare const tools: { view_image(args: {\n // Local filesystem path to an image file.\n path: string;\n}): Promise<{\n // Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved.\n detail: \"high\" | \"original\";\n // Data URL for the loaded image.\n image_url: string;\n}>; };\n```",
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_exports_all_tools_metadata_for_namespaced_mcp_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const tool = ALL_TOOLS.find(
|
|
({ name }) => name === "mcp__rmcp__echo"
|
|
);
|
|
text(JSON.stringify(tool));
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn_with_rmcp(&server, "use exec to inspect ALL_TOOLS", code).await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec ALL_TOOLS MCP lookup failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(
|
|
&custom_tool_output_last_non_empty_text(&req, "call-1")
|
|
.expect("exec ALL_TOOLS MCP lookup should emit JSON"),
|
|
)?;
|
|
assert_eq!(
|
|
parsed,
|
|
serde_json::json!({
|
|
"name": "mcp__rmcp__echo",
|
|
"description": concat!(
|
|
"Use these tools to exercise the rmcp test server.\n\n",
|
|
"Echo back the provided message and include environment data.\n\n",
|
|
"exec tool declaration:\n",
|
|
"```ts\n",
|
|
"declare const tools: { mcp__rmcp__echo(args: { env_var?: string; message: string; }): ",
|
|
"Promise<CallToolResult<{ echo: string; env: string | null; }>>; };\n",
|
|
"```",
|
|
),
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_call_hidden_dynamic_tools() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let base_test = builder.build(&server).await?;
|
|
let new_thread = base_test
|
|
.thread_manager
|
|
.start_thread_with_tools(
|
|
base_test.config.clone(),
|
|
vec![DynamicToolSpec::Namespace(DynamicToolNamespaceSpec {
|
|
name: "codex_app".to_string(),
|
|
description: "Codex app tools.".to_string(),
|
|
tools: vec![DynamicToolNamespaceTool::Function(
|
|
DynamicToolFunctionSpec {
|
|
name: "hidden_dynamic_tool".to_string(),
|
|
description: "A hidden dynamic tool.".to_string(),
|
|
input_schema: serde_json::json!({
|
|
"type": "object",
|
|
"properties": {
|
|
"city": { "type": "string" }
|
|
},
|
|
"required": ["city"],
|
|
"additionalProperties": false,
|
|
}),
|
|
defer_loading: true,
|
|
},
|
|
)],
|
|
})],
|
|
)
|
|
.await?;
|
|
let mut test = base_test;
|
|
test.codex = new_thread.thread;
|
|
test.session_configured = new_thread.session_configured;
|
|
|
|
let code = r#"
|
|
const tool = ALL_TOOLS.find(({ name }) => name === "codex_app__hidden_dynamic_tool");
|
|
const out = await tools.codex_app__hidden_dynamic_tool({ city: "Paris" });
|
|
text(
|
|
JSON.stringify({
|
|
name: tool?.name ?? null,
|
|
description: tool?.description ?? null,
|
|
out,
|
|
})
|
|
);
|
|
"#;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call("call-1", "exec", code),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
let cwd = test.config.cwd.clone();
|
|
let (sandbox_policy, permission_profile) =
|
|
turn_permission_fields(PermissionProfile::Disabled, cwd.as_path());
|
|
|
|
test.codex
|
|
.submit(Op::UserInput {
|
|
items: vec![UserInput::Text {
|
|
text: "use exec to inspect and call hidden tools".into(),
|
|
text_elements: Vec::new(),
|
|
}],
|
|
final_output_json_schema: None,
|
|
responsesapi_client_metadata: None,
|
|
additional_context: Default::default(),
|
|
thread_settings: codex_protocol::protocol::ThreadSettingsOverrides {
|
|
environments: Some(codex_protocol::protocol::TurnEnvironmentSelections::new(
|
|
cwd,
|
|
Vec::new(),
|
|
)),
|
|
approval_policy: Some(AskForApproval::Never),
|
|
sandbox_policy: Some(sandbox_policy),
|
|
permission_profile,
|
|
collaboration_mode: Some(codex_protocol::config_types::CollaborationMode {
|
|
mode: codex_protocol::config_types::ModeKind::Default,
|
|
settings: codex_protocol::config_types::Settings {
|
|
model: test.session_configured.model.clone(),
|
|
reasoning_effort: None,
|
|
developer_instructions: None,
|
|
},
|
|
}),
|
|
..Default::default()
|
|
},
|
|
})
|
|
.await?;
|
|
|
|
let turn_id = wait_for_event_match(&test.codex, |event| match event {
|
|
EventMsg::TurnStarted(event) => Some(event.turn_id.clone()),
|
|
_ => None,
|
|
})
|
|
.await;
|
|
let request = wait_for_event_match(&test.codex, |event| match event {
|
|
EventMsg::DynamicToolCallRequest(request) => Some(request.clone()),
|
|
_ => None,
|
|
})
|
|
.await;
|
|
assert_eq!(request.namespace.as_deref(), Some("codex_app"));
|
|
assert_eq!(request.tool, "hidden_dynamic_tool");
|
|
assert_eq!(request.arguments, serde_json::json!({ "city": "Paris" }));
|
|
test.codex
|
|
.submit(Op::DynamicToolResponse {
|
|
id: request.call_id,
|
|
response: DynamicToolResponse {
|
|
content_items: vec![DynamicToolCallOutputContentItem::InputText {
|
|
text: "hidden-ok".to_string(),
|
|
}],
|
|
success: true,
|
|
},
|
|
})
|
|
.await?;
|
|
wait_for_event(&test.codex, |event| match event {
|
|
EventMsg::TurnComplete(event) => event.turn_id == turn_id,
|
|
_ => false,
|
|
})
|
|
.await;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec hidden dynamic tool call failed unexpectedly: {output}"
|
|
);
|
|
|
|
let parsed: Value = serde_json::from_str(
|
|
&custom_tool_output_last_non_empty_text(&req, "call-1")
|
|
.expect("exec hidden dynamic tool lookup should emit JSON"),
|
|
)?;
|
|
assert_eq!(
|
|
parsed.get("name"),
|
|
Some(&Value::String("codex_app__hidden_dynamic_tool".to_string()))
|
|
);
|
|
assert_eq!(
|
|
parsed.get("out"),
|
|
Some(&Value::String("hidden-ok".to_string()))
|
|
);
|
|
assert!(
|
|
parsed
|
|
.get("description")
|
|
.and_then(Value::as_str)
|
|
.is_some_and(|description| {
|
|
description.contains("Codex app tools.")
|
|
&& description.contains("A hidden dynamic tool.")
|
|
&& description.contains("declare const tools:")
|
|
&& description.contains("codex_app__hidden_dynamic_tool(args:")
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_excludes_configured_nested_tool_namespaces() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(|config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
config.code_mode.excluded_tool_namespaces = vec!["excluded".to_string()];
|
|
});
|
|
let base_test = builder.build(&server).await?;
|
|
let new_thread = base_test
|
|
.thread_manager
|
|
.start_thread_with_tools(
|
|
base_test.config.clone(),
|
|
vec![DynamicToolSpec::Namespace(DynamicToolNamespaceSpec {
|
|
name: "excluded".to_string(),
|
|
description: "Excluded tools.".to_string(),
|
|
tools: vec![DynamicToolNamespaceTool::Function(
|
|
DynamicToolFunctionSpec {
|
|
name: "lookup".to_string(),
|
|
description: "An excluded dynamic tool.".to_string(),
|
|
input_schema: serde_json::json!({
|
|
"type": "object",
|
|
"properties": {},
|
|
"additionalProperties": false,
|
|
}),
|
|
defer_loading: false,
|
|
},
|
|
)],
|
|
})],
|
|
)
|
|
.await?;
|
|
let mut test = base_test;
|
|
test.codex = new_thread.thread;
|
|
test.session_configured = new_thread.session_configured;
|
|
|
|
let first_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
text(JSON.stringify({
|
|
excludedType: typeof tools.excluded__lookup,
|
|
excludedMetadata: ALL_TOOLS.some(({ name }) => name === "excluded__lookup"),
|
|
allowedType: typeof tools.update_plan,
|
|
allowedMetadata: ALL_TOOLS.some(({ name }) => name === "update_plan"),
|
|
}));
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_mock = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("use exec to inspect nested tool namespaces")
|
|
.await?;
|
|
|
|
assert!(
|
|
tool_names(&first_mock.single_request().body_json()).contains(&"excluded".to_string()),
|
|
"excluded namespace should remain directly exposed in mixed code mode"
|
|
);
|
|
let request = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&request, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec configured namespace exclusion failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(
|
|
serde_json::from_str::<Value>(&output)?,
|
|
serde_json::json!({
|
|
"excludedType": "undefined",
|
|
"excludedMetadata": false,
|
|
"allowedType": "function",
|
|
"allowedMetadata": true,
|
|
})
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_print_content_only_mcp_tool_result_fields() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const { content, structuredContent, isError } = await tools.mcp__rmcp__image_scenario({
|
|
scenario: "text_only",
|
|
caption: "caption from mcp",
|
|
});
|
|
text(
|
|
`firstType=${content[0]?.type ?? "missing"}\n` +
|
|
`firstText=${content[0]?.text ?? "missing"}\n` +
|
|
`structuredContent=${String(structuredContent ?? null)}\n` +
|
|
`isError=${String(isError)}`
|
|
);
|
|
"#;
|
|
|
|
let (_test, second_mock) = run_code_mode_turn_with_rmcp(
|
|
&server,
|
|
"use exec to run the rmcp image scenario tool",
|
|
code,
|
|
)
|
|
.await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec rmcp image scenario call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(
|
|
output,
|
|
"firstType=text
|
|
firstText=caption from mcp
|
|
structuredContent=null
|
|
isError=false"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_print_error_mcp_tool_result_fields() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let code = r#"
|
|
const { content, structuredContent, isError } = await tools.mcp__rmcp__echo({});
|
|
const firstText = content[0]?.text ?? "";
|
|
const mentionsMissingMessage =
|
|
firstText.includes("missing field") && firstText.includes("message");
|
|
text(
|
|
`isError=${String(isError)}\n` +
|
|
`contentLength=${content.length}\n` +
|
|
`mentionsMissingMessage=${String(mentionsMissingMessage)}\n` +
|
|
`structuredContent=${String(structuredContent ?? null)}`
|
|
);
|
|
"#;
|
|
|
|
let (_test, second_mock) =
|
|
run_code_mode_turn_with_rmcp(&server, "use exec to call rmcp echo badly", code).await?;
|
|
|
|
let req = second_mock.single_request();
|
|
let (output, success) = custom_tool_output_body_and_success(&req, "call-1");
|
|
assert_ne!(
|
|
success,
|
|
Some(false),
|
|
"exec rmcp error call failed unexpectedly: {output}"
|
|
);
|
|
assert_eq!(
|
|
output,
|
|
"isError=true
|
|
contentLength=1
|
|
mentionsMissingMessage=true
|
|
structuredContent=null"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_store_and_load_values_across_turns() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let mut builder = test_codex().with_config(move |config| {
|
|
let _ = config.features.enable(Feature::CodeMode);
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_custom_tool_call(
|
|
"call-1",
|
|
"exec",
|
|
r#"
|
|
store("nb", { title: "Notebook", items: [1, true, null] });
|
|
text("stored");
|
|
"#,
|
|
),
|
|
ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let first_follow_up = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-1", "stored"),
|
|
ev_completed("resp-2"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("store value for later").await?;
|
|
|
|
let first_request = first_follow_up.single_request();
|
|
let (first_output, first_success) =
|
|
custom_tool_output_body_and_success(&first_request, "call-1");
|
|
assert_ne!(
|
|
first_success,
|
|
Some(false),
|
|
"exec store call failed unexpectedly: {first_output}"
|
|
);
|
|
assert_eq!(first_output, "stored");
|
|
|
|
responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_response_created("resp-3"),
|
|
ev_custom_tool_call(
|
|
"call-2",
|
|
"exec",
|
|
r#"
|
|
text(JSON.stringify(load("nb")));
|
|
"#,
|
|
),
|
|
ev_completed("resp-3"),
|
|
]),
|
|
)
|
|
.await;
|
|
let second_follow_up = responses::mount_sse_once(
|
|
&server,
|
|
sse(vec![
|
|
ev_assistant_message("msg-2", "loaded"),
|
|
ev_completed("resp-4"),
|
|
]),
|
|
)
|
|
.await;
|
|
|
|
test.submit_turn("load the stored value").await?;
|
|
|
|
let second_request = second_follow_up.single_request();
|
|
let (second_output, second_success) =
|
|
custom_tool_output_body_and_success(&second_request, "call-2");
|
|
assert_ne!(
|
|
second_success,
|
|
Some(false),
|
|
"exec load call failed unexpectedly: {second_output}"
|
|
);
|
|
let loaded: Value = serde_json::from_str(
|
|
&custom_tool_output_last_non_empty_text(&second_request, "call-2")
|
|
.expect("exec load call should emit JSON"),
|
|
)?;
|
|
assert_eq!(
|
|
loaded,
|
|
serde_json::json!({ "title": "Notebook", "items": [1, true, null] })
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn code_mode_can_compare_elapsed_time_around_set_timeout() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let (_test, second_mock) = run_code_mode_turn(
|
|
&server,
|
|
"measure elapsed time around setTimeout",
|
|
r#"
|
|
const start_ms = Date.now();
|
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
const end_ms = Date.now();
|
|
text(JSON.stringify({
|
|
start_ms,
|
|
end_ms,
|
|
elapsed_ms: end_ms - start_ms,
|
|
waited_long_enough: end_ms - start_ms >= 100,
|
|
}));
|
|
"#,
|
|
)
|
|
.await?;
|
|
|
|
let second_request = second_mock.single_request();
|
|
let (second_output, second_success) =
|
|
custom_tool_output_body_and_success(&second_request, "call-1");
|
|
assert_ne!(
|
|
second_success,
|
|
Some(false),
|
|
"exec compare time call failed unexpectedly: {second_output}"
|
|
);
|
|
let compared: Value = serde_json::from_str(
|
|
&custom_tool_output_last_non_empty_text(&second_request, "call-1")
|
|
.expect("exec compare time call should emit JSON"),
|
|
)?;
|
|
let elapsed_ms = compared
|
|
.get("elapsed_ms")
|
|
.and_then(Value::as_i64)
|
|
.expect("elapsed_ms should be an integer");
|
|
assert!(
|
|
elapsed_ms >= 100,
|
|
"expected elapsed_ms >= 100, got {elapsed_ms}"
|
|
);
|
|
assert_eq!(compared.get("waited_long_enough"), Some(&Value::Bool(true)));
|
|
|
|
Ok(())
|
|
}
|