mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
fix(linux-sandbox): preserve shell cleanup on interruption (#22729)
## Why Interrupted `shell_command` calls can race with the outer tool-dispatch cancellation path. When that happens, the runtime future may be dropped before the spawned process gets a chance to run `SIGTERM` cleanup. For bwrapd-backed Linux sandbox commands, that can leave synthetic protected-path mount bookkeeping such as `.git/.codex` registrations under `/tmp` behind after a TUI interruption. The relevant cancellation points are the outer dispatch race in [`core/src/tools/parallel.rs`](https://github.com/openai/codex/blob/bd184ba84703cc924921ed883f0cf17d3dba60ff/codex-rs/core/src/tools/parallel.rs#L91-L132) and the process shutdown logic in [`core/src/exec.rs`](https://github.com/openai/codex/blob/bd184ba84703cc924921ed883f0cf17d3dba60ff/codex-rs/core/src/exec.rs#L1367-L1393). ## What changed - Keep `shell_command` dispatch alive long enough for the runtime to finish cancellation cleanup instead of immediately returning the synthetic aborted response. - Fold shell-turn cancellation into the existing `ExecExpiration` path in [`core/src/tools/runtimes/shell.rs`](https://github.com/openai/codex/blob/bd184ba84703cc924921ed883f0cf17d3dba60ff/codex-rs/core/src/tools/runtimes/shell.rs#L267-L274), so cancellation and timeout behavior stay centralized. - On cancellation, send `SIGTERM` first, wait briefly for cleanup to run, then hard-kill any remaining descendants in the original process group. - Treat `ESRCH` as an already-gone process-group cleanup case in `codex-utils-pty`, which keeps best-effort teardown from surfacing a stale-process race as an error. ## Verification - `cargo test -p codex-core cancellation` - Added regression coverage for: - `shell_tool_cancellation_waits_for_runtime_cleanup` - `process_exec_tool_call_cancellation_allows_sigterm_cleanup`
This commit is contained in:
committed by
GitHub
Unverified
parent
07a930138f
commit
9152ebd289
@@ -2,6 +2,7 @@ use codex_features::Feature;
|
||||
use codex_protocol::models::ShellCommandToolCallParams;
|
||||
use serde_json::Value as JsonValue;
|
||||
use std::sync::Arc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::exec::ExecParams;
|
||||
use crate::exec_policy::ExecApprovalRequest;
|
||||
@@ -44,6 +45,7 @@ fn shell_command_payload_command(payload: &ToolPayload) -> Option<String> {
|
||||
struct RunExecLikeArgs {
|
||||
tool_name: ToolName,
|
||||
exec_params: ExecParams,
|
||||
cancellation_token: CancellationToken,
|
||||
hook_command: String,
|
||||
shell_type: Option<ShellType>,
|
||||
additional_permissions: Option<AdditionalPermissionProfile>,
|
||||
@@ -59,6 +61,7 @@ async fn run_exec_like(args: RunExecLikeArgs) -> Result<FunctionToolOutput, Func
|
||||
let RunExecLikeArgs {
|
||||
tool_name,
|
||||
exec_params,
|
||||
cancellation_token,
|
||||
hook_command,
|
||||
shell_type,
|
||||
additional_permissions,
|
||||
@@ -183,6 +186,7 @@ async fn run_exec_like(args: RunExecLikeArgs) -> Result<FunctionToolOutput, Func
|
||||
hook_command,
|
||||
cwd: exec_params.cwd.clone(),
|
||||
timeout_ms: exec_params.expiration.timeout_ms(),
|
||||
cancellation_token,
|
||||
env: exec_params.env.clone(),
|
||||
explicit_env_overrides,
|
||||
network: exec_params.network.clone(),
|
||||
|
||||
@@ -148,6 +148,7 @@ impl ToolExecutor<ToolInvocation> for ShellCommandHandler {
|
||||
let ToolInvocation {
|
||||
session,
|
||||
turn,
|
||||
cancellation_token,
|
||||
tracker,
|
||||
call_id,
|
||||
payload,
|
||||
@@ -185,6 +186,7 @@ impl ToolExecutor<ToolInvocation> for ShellCommandHandler {
|
||||
run_exec_like(RunExecLikeArgs {
|
||||
tool_name,
|
||||
exec_params,
|
||||
cancellation_token,
|
||||
hook_command: params.command,
|
||||
shell_type,
|
||||
additional_permissions: params.additional_permissions.clone(),
|
||||
@@ -205,6 +207,10 @@ impl CoreToolRuntime for ShellCommandHandler {
|
||||
matches!(payload, ToolPayload::Function { .. })
|
||||
}
|
||||
|
||||
fn waits_for_runtime_cancellation(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn pre_tool_use_payload(&self, invocation: &ToolInvocation) -> Option<PreToolUsePayload> {
|
||||
shell_command_payload_command(&invocation.payload).map(|command| PreToolUsePayload {
|
||||
tool_name: HookToolName::bash(),
|
||||
|
||||
@@ -92,6 +92,7 @@ impl ToolCallRuntime {
|
||||
let tracker = Arc::clone(&self.tracker);
|
||||
let lock = Arc::clone(&self.parallel_execution);
|
||||
let invocation_cancellation_token = cancellation_token.clone();
|
||||
let wait_for_runtime_cancellation = self.router.tool_waits_for_runtime_cancellation(&call);
|
||||
let started = Instant::now();
|
||||
let abort_session = Arc::clone(&session);
|
||||
let abort_source = source.clone();
|
||||
@@ -140,23 +141,35 @@ impl ToolCallRuntime {
|
||||
} else {
|
||||
let secs = started.elapsed().as_secs_f32().max(0.1);
|
||||
abort_dispatch_span.record("aborted", true);
|
||||
handle.abort();
|
||||
match handle.await {
|
||||
Ok(result) => result,
|
||||
Err(err) if err.is_cancelled() => {
|
||||
let response = Self::aborted_response(&call, secs);
|
||||
notify_tool_aborted(
|
||||
abort_session.as_ref(),
|
||||
abort_turn.as_ref(),
|
||||
call.call_id.as_str(),
|
||||
&call.tool_name,
|
||||
abort_source,
|
||||
)
|
||||
.await;
|
||||
Ok(response)
|
||||
if wait_for_runtime_cancellation {
|
||||
if terminal_outcome_reached.swap(true, Ordering::AcqRel) {
|
||||
return handle.await.map_err(Self::tool_task_join_error)?;
|
||||
}
|
||||
// The abort owns the terminal outcome; await only so
|
||||
// the runtime can finish process teardown.
|
||||
match handle.await {
|
||||
Ok(_) => {}
|
||||
Err(err) if err.is_cancelled() => {}
|
||||
Err(err) => return Err(Self::tool_task_join_error(err)),
|
||||
}
|
||||
} else {
|
||||
handle.abort();
|
||||
match handle.await {
|
||||
Ok(result) => return result,
|
||||
Err(err) if err.is_cancelled() => {}
|
||||
Err(err) => return Err(Self::tool_task_join_error(err)),
|
||||
}
|
||||
Err(err) => Err(Self::tool_task_join_error(err)),
|
||||
}
|
||||
let response = Self::aborted_response(&call, secs);
|
||||
notify_tool_aborted(
|
||||
abort_session.as_ref(),
|
||||
abort_turn.as_ref(),
|
||||
call.call_id.as_str(),
|
||||
&call.tool_name,
|
||||
abort_source,
|
||||
)
|
||||
.await;
|
||||
Ok(response)
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -274,6 +287,85 @@ mod tests {
|
||||
|
||||
impl CoreToolRuntime for ImmediateHandler {}
|
||||
|
||||
struct CancellationCleanupHandler {
|
||||
tool_name: codex_tools::ToolName,
|
||||
started: std::sync::Mutex<Option<oneshot::Sender<()>>>,
|
||||
cleanup_started: std::sync::Mutex<Option<oneshot::Sender<()>>>,
|
||||
allow_cleanup: Arc<Notify>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ToolExecutor<ToolInvocation> for CancellationCleanupHandler {
|
||||
fn tool_name(&self) -> codex_tools::ToolName {
|
||||
self.tool_name.clone()
|
||||
}
|
||||
|
||||
fn spec(&self) -> codex_tools::ToolSpec {
|
||||
codex_tools::ToolSpec::Function(codex_tools::ResponsesApiTool {
|
||||
name: self.tool_name.name.clone(),
|
||||
description: "Cancellation cleanup test tool.".to_string(),
|
||||
strict: false,
|
||||
defer_loading: None,
|
||||
parameters: codex_tools::JsonSchema::default(),
|
||||
output_schema: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn handle(
|
||||
&self,
|
||||
invocation: ToolInvocation,
|
||||
) -> Result<Box<dyn crate::tools::context::ToolOutput>, FunctionCallError> {
|
||||
let started = self
|
||||
.started
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.take();
|
||||
if let Some(started) = started {
|
||||
let _ = started.send(());
|
||||
}
|
||||
invocation.cancellation_token.cancelled().await;
|
||||
let cleanup_started = self
|
||||
.cleanup_started
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.take();
|
||||
if let Some(cleanup_started) = cleanup_started {
|
||||
let _ = cleanup_started.send(());
|
||||
}
|
||||
self.allow_cleanup.notified().await;
|
||||
Ok(Box::new(FunctionToolOutput::from_text(
|
||||
"cleanup complete".to_string(),
|
||||
Some(false),
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
impl CoreToolRuntime for CancellationCleanupHandler {
|
||||
fn waits_for_runtime_cancellation(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
struct FinishRecorder {
|
||||
records: Arc<std::sync::Mutex<Vec<ToolCallOutcome>>>,
|
||||
}
|
||||
|
||||
impl codex_extension_api::ToolLifecycleContributor for FinishRecorder {
|
||||
fn on_tool_finish<'a>(
|
||||
&'a self,
|
||||
input: codex_extension_api::ToolFinishInput<'a>,
|
||||
) -> codex_extension_api::ToolLifecycleFuture<'a> {
|
||||
let records = Arc::clone(&self.records);
|
||||
let outcome = input.outcome;
|
||||
Box::pin(async move {
|
||||
records
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.push(outcome);
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct BlockingFinishContributor {
|
||||
records: Arc<std::sync::Mutex<Vec<ToolCallOutcome>>>,
|
||||
finish_started: std::sync::Mutex<Option<oneshot::Sender<()>>>,
|
||||
@@ -375,4 +467,75 @@ mod tests {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_waiting_for_runtime_cleanup_emits_only_aborted_lifecycle()
|
||||
-> anyhow::Result<()> {
|
||||
let (mut session, turn_context) = crate::session::tests::make_session_and_context().await;
|
||||
let records = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let mut builder =
|
||||
codex_extension_api::ExtensionRegistryBuilder::<crate::config::Config>::new();
|
||||
builder.tool_lifecycle_contributor(Arc::new(FinishRecorder {
|
||||
records: Arc::clone(&records),
|
||||
}));
|
||||
session.services.extensions = Arc::new(builder.build());
|
||||
|
||||
let session = Arc::new(session);
|
||||
let turn_context = Arc::new(turn_context);
|
||||
let tool_name = codex_tools::ToolName::plain("cleanup_tool");
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (cleanup_started_tx, cleanup_started_rx) = oneshot::channel();
|
||||
let allow_cleanup = Arc::new(Notify::new());
|
||||
let handler = Arc::new(CancellationCleanupHandler {
|
||||
tool_name: tool_name.clone(),
|
||||
started: std::sync::Mutex::new(Some(started_tx)),
|
||||
cleanup_started: std::sync::Mutex::new(Some(cleanup_started_tx)),
|
||||
allow_cleanup: Arc::clone(&allow_cleanup),
|
||||
}) as Arc<dyn CoreToolRuntime>;
|
||||
let router = Arc::new(ToolRouter::from_parts(
|
||||
ToolRegistry::from_tools([handler]),
|
||||
Vec::new(),
|
||||
));
|
||||
let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new()));
|
||||
let runtime = ToolCallRuntime::new(router, session, turn_context, tracker);
|
||||
let cancellation_token = CancellationToken::new();
|
||||
let call = ToolCall {
|
||||
tool_name,
|
||||
call_id: "call-1".to_string(),
|
||||
payload: ToolPayload::Function {
|
||||
arguments: "{}".to_string(),
|
||||
},
|
||||
};
|
||||
|
||||
let response_task =
|
||||
tokio::spawn(runtime.handle_tool_call(call, cancellation_token.clone()));
|
||||
started_rx.await.expect("handler should start");
|
||||
cancellation_token.cancel();
|
||||
cleanup_started_rx
|
||||
.await
|
||||
.expect("handler should start cleanup");
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
allow_cleanup.notify_one();
|
||||
|
||||
let response = tokio::time::timeout(Duration::from_secs(1), response_task)
|
||||
.await
|
||||
.expect("timed out waiting for tool response")
|
||||
.expect("tool response task should join")?;
|
||||
let ResponseInputItem::FunctionCallOutput { output, .. } = response else {
|
||||
anyhow::bail!("cancelled tool should return function output");
|
||||
};
|
||||
let FunctionCallOutputBody::Text(text) = output.body else {
|
||||
anyhow::bail!("cancelled tool output should be text");
|
||||
};
|
||||
assert!(text.contains("aborted by user"));
|
||||
|
||||
let actual = records
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.drain(..)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(vec![ToolCallOutcome::Aborted], actual);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,6 +57,12 @@ pub(crate) trait CoreToolRuntime: ToolExecutor<ToolInvocation> {
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether cancellation should let the handler finish teardown before the
|
||||
/// host returns an aborted tool response.
|
||||
fn waits_for_runtime_cancellation(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn telemetry_tags<'a>(
|
||||
&'a self,
|
||||
_invocation: &'a ToolInvocation,
|
||||
@@ -284,6 +290,10 @@ impl CoreToolRuntime for ExposureOverride {
|
||||
self.handler.matches_kind(payload)
|
||||
}
|
||||
|
||||
fn waits_for_runtime_cancellation(&self) -> bool {
|
||||
self.handler.waits_for_runtime_cancellation()
|
||||
}
|
||||
|
||||
fn pre_tool_use_payload(&self, invocation: &ToolInvocation) -> Option<PreToolUsePayload> {
|
||||
self.handler.pre_tool_use_payload(invocation)
|
||||
}
|
||||
@@ -381,6 +391,11 @@ impl ToolRegistry {
|
||||
Some(tool.supports_parallel_tool_calls())
|
||||
}
|
||||
|
||||
pub(crate) fn waits_for_runtime_cancellation(&self, name: &ToolName) -> Option<bool> {
|
||||
let tool = self.tool(name)?;
|
||||
Some(tool.waits_for_runtime_cancellation())
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub(crate) async fn dispatch_any(
|
||||
&self,
|
||||
@@ -497,10 +512,12 @@ impl ToolRegistry {
|
||||
PreToolUseHookResult::Blocked(message) => {
|
||||
let err = FunctionCallError::RespondToModel(message);
|
||||
dispatch_trace.record_failed(&err);
|
||||
if let Some(terminal_outcome_reached) = &terminal_outcome_reached {
|
||||
terminal_outcome_reached.store(true, Ordering::Release);
|
||||
}
|
||||
notify_tool_finish(&invocation, ToolCallOutcome::Blocked).await;
|
||||
notify_tool_finish_if_unclaimed(
|
||||
&invocation,
|
||||
terminal_outcome_reached.as_deref(),
|
||||
ToolCallOutcome::Blocked,
|
||||
)
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
PreToolUseHookResult::Continue {
|
||||
@@ -511,11 +528,9 @@ impl ToolRegistry {
|
||||
}
|
||||
Err(err) => {
|
||||
dispatch_trace.record_failed(&err);
|
||||
if let Some(terminal_outcome_reached) = &terminal_outcome_reached {
|
||||
terminal_outcome_reached.store(true, Ordering::Release);
|
||||
}
|
||||
notify_tool_finish(
|
||||
notify_tool_finish_if_unclaimed(
|
||||
&invocation,
|
||||
terminal_outcome_reached.as_deref(),
|
||||
ToolCallOutcome::Failed {
|
||||
handler_executed: false,
|
||||
},
|
||||
@@ -638,18 +653,21 @@ impl ToolRegistry {
|
||||
handler_executed: true,
|
||||
},
|
||||
};
|
||||
if let Some(terminal_outcome_reached) = &terminal_outcome_reached {
|
||||
terminal_outcome_reached.store(true, Ordering::Release);
|
||||
}
|
||||
notify_tool_finish(&invocation, lifecycle_outcome).await;
|
||||
let finished = notify_tool_finish_if_unclaimed(
|
||||
&invocation,
|
||||
terminal_outcome_reached.as_deref(),
|
||||
lifecycle_outcome,
|
||||
)
|
||||
.await;
|
||||
|
||||
if let Err(err) = invocation
|
||||
.session
|
||||
.goal_runtime_apply(GoalRuntimeEvent::ToolCompleted {
|
||||
turn_context: invocation.turn.as_ref(),
|
||||
tool_name: tool_name.name.as_str(),
|
||||
})
|
||||
.await
|
||||
if finished
|
||||
&& let Err(err) = invocation
|
||||
.session
|
||||
.goal_runtime_apply(GoalRuntimeEvent::ToolCompleted {
|
||||
turn_context: invocation.turn.as_ref(),
|
||||
tool_name: tool_name.name.as_str(),
|
||||
})
|
||||
.await
|
||||
{
|
||||
warn!("failed to account thread goal progress after tool call: {err}");
|
||||
}
|
||||
@@ -676,6 +694,19 @@ impl ToolRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
async fn notify_tool_finish_if_unclaimed(
|
||||
invocation: &ToolInvocation,
|
||||
terminal_outcome_reached: Option<&AtomicBool>,
|
||||
outcome: ToolCallOutcome,
|
||||
) -> bool {
|
||||
if terminal_outcome_reached.is_some_and(|reached| reached.swap(true, Ordering::AcqRel)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
notify_tool_finish(invocation, outcome).await;
|
||||
true
|
||||
}
|
||||
|
||||
async fn handle_any_tool(
|
||||
tool: &dyn CoreToolRuntime,
|
||||
invocation: ToolInvocation,
|
||||
|
||||
@@ -86,6 +86,12 @@ impl ToolRouter {
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
pub fn tool_waits_for_runtime_cancellation(&self, call: &ToolCall) -> bool {
|
||||
self.registry
|
||||
.waits_for_runtime_cancellation(&call.tool_name)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip_all, err)]
|
||||
pub fn build_tool_call(item: ResponseItem) -> Result<Option<ToolCall>, FunctionCallError> {
|
||||
match item {
|
||||
|
||||
@@ -44,6 +44,7 @@ use codex_shell_command::powershell::prefix_powershell_script_with_utf8;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use futures::future::BoxFuture;
|
||||
use std::collections::HashMap;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ShellRequest {
|
||||
@@ -52,6 +53,7 @@ pub struct ShellRequest {
|
||||
pub hook_command: String,
|
||||
pub cwd: AbsolutePathBuf,
|
||||
pub timeout_ms: Option<u64>,
|
||||
pub cancellation_token: CancellationToken,
|
||||
pub env: HashMap<String, String>,
|
||||
pub explicit_env_overrides: HashMap<String, String>,
|
||||
pub network: Option<NetworkProxy>,
|
||||
@@ -265,6 +267,7 @@ impl ToolRuntime<ShellRequest, ExecToolCallOutput> for ShellRuntime {
|
||||
let command =
|
||||
build_sandbox_command(&command, &req.cwd, &env, req.additional_permissions.clone())?;
|
||||
let mut expiration: crate::exec::ExecExpiration = req.timeout_ms.into();
|
||||
expiration = expiration.with_cancellation(req.cancellation_token.clone());
|
||||
if let Some(cancellation) = attempt.network_denial_cancellation_token.clone() {
|
||||
expiration = expiration.with_cancellation(cancellation);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user