fix(linux-sandbox): preserve shell cleanup on interruption (#22729)

## Why
Interrupted `shell_command` calls can race with the outer tool-dispatch
cancellation path. When that happens, the runtime future may be dropped
before the spawned process gets a chance to run `SIGTERM` cleanup. For
bwrapd-backed Linux sandbox commands, that can leave synthetic
protected-path mount bookkeeping such as `.git/.codex` registrations
under `/tmp` behind after a TUI interruption.

The relevant cancellation points are the outer dispatch race in
[`core/src/tools/parallel.rs`](https://github.com/openai/codex/blob/bd184ba84703cc924921ed883f0cf17d3dba60ff/codex-rs/core/src/tools/parallel.rs#L91-L132)
and the process shutdown logic in
[`core/src/exec.rs`](https://github.com/openai/codex/blob/bd184ba84703cc924921ed883f0cf17d3dba60ff/codex-rs/core/src/exec.rs#L1367-L1393).

## What changed
- Keep `shell_command` dispatch alive long enough for the runtime to
finish cancellation cleanup instead of immediately returning the
synthetic aborted response.
- Fold shell-turn cancellation into the existing `ExecExpiration` path
in
[`core/src/tools/runtimes/shell.rs`](https://github.com/openai/codex/blob/bd184ba84703cc924921ed883f0cf17d3dba60ff/codex-rs/core/src/tools/runtimes/shell.rs#L267-L274),
so cancellation and timeout behavior stay centralized.
- On cancellation, send `SIGTERM` first, wait briefly for cleanup to
run, then hard-kill any remaining descendants in the original process
group.
- Treat `ESRCH` as an already-gone process-group cleanup case in
`codex-utils-pty`, which keeps best-effort teardown from surfacing a
stale-process race as an error.

## Verification
- `cargo test -p codex-core cancellation`
- Added regression coverage for:
  - `shell_tool_cancellation_waits_for_runtime_cleanup`
  - `process_exec_tool_call_cancellation_allows_sigterm_cleanup`
This commit is contained in:
viyatb-oai
2026-05-27 12:59:11 -07:00
committed by GitHub
Unverified
parent 07a930138f
commit 9152ebd289
10 changed files with 474 additions and 47 deletions
@@ -2,6 +2,7 @@ use codex_features::Feature;
use codex_protocol::models::ShellCommandToolCallParams;
use serde_json::Value as JsonValue;
use std::sync::Arc;
use tokio_util::sync::CancellationToken;
use crate::exec::ExecParams;
use crate::exec_policy::ExecApprovalRequest;
@@ -44,6 +45,7 @@ fn shell_command_payload_command(payload: &ToolPayload) -> Option<String> {
struct RunExecLikeArgs {
tool_name: ToolName,
exec_params: ExecParams,
cancellation_token: CancellationToken,
hook_command: String,
shell_type: Option<ShellType>,
additional_permissions: Option<AdditionalPermissionProfile>,
@@ -59,6 +61,7 @@ async fn run_exec_like(args: RunExecLikeArgs) -> Result<FunctionToolOutput, Func
let RunExecLikeArgs {
tool_name,
exec_params,
cancellation_token,
hook_command,
shell_type,
additional_permissions,
@@ -183,6 +186,7 @@ async fn run_exec_like(args: RunExecLikeArgs) -> Result<FunctionToolOutput, Func
hook_command,
cwd: exec_params.cwd.clone(),
timeout_ms: exec_params.expiration.timeout_ms(),
cancellation_token,
env: exec_params.env.clone(),
explicit_env_overrides,
network: exec_params.network.clone(),
@@ -148,6 +148,7 @@ impl ToolExecutor<ToolInvocation> for ShellCommandHandler {
let ToolInvocation {
session,
turn,
cancellation_token,
tracker,
call_id,
payload,
@@ -185,6 +186,7 @@ impl ToolExecutor<ToolInvocation> for ShellCommandHandler {
run_exec_like(RunExecLikeArgs {
tool_name,
exec_params,
cancellation_token,
hook_command: params.command,
shell_type,
additional_permissions: params.additional_permissions.clone(),
@@ -205,6 +207,10 @@ impl CoreToolRuntime for ShellCommandHandler {
matches!(payload, ToolPayload::Function { .. })
}
fn waits_for_runtime_cancellation(&self) -> bool {
true
}
fn pre_tool_use_payload(&self, invocation: &ToolInvocation) -> Option<PreToolUsePayload> {
shell_command_payload_command(&invocation.payload).map(|command| PreToolUsePayload {
tool_name: HookToolName::bash(),
+178 -15
View File
@@ -92,6 +92,7 @@ impl ToolCallRuntime {
let tracker = Arc::clone(&self.tracker);
let lock = Arc::clone(&self.parallel_execution);
let invocation_cancellation_token = cancellation_token.clone();
let wait_for_runtime_cancellation = self.router.tool_waits_for_runtime_cancellation(&call);
let started = Instant::now();
let abort_session = Arc::clone(&session);
let abort_source = source.clone();
@@ -140,23 +141,35 @@ impl ToolCallRuntime {
} else {
let secs = started.elapsed().as_secs_f32().max(0.1);
abort_dispatch_span.record("aborted", true);
handle.abort();
match handle.await {
Ok(result) => result,
Err(err) if err.is_cancelled() => {
let response = Self::aborted_response(&call, secs);
notify_tool_aborted(
abort_session.as_ref(),
abort_turn.as_ref(),
call.call_id.as_str(),
&call.tool_name,
abort_source,
)
.await;
Ok(response)
if wait_for_runtime_cancellation {
if terminal_outcome_reached.swap(true, Ordering::AcqRel) {
return handle.await.map_err(Self::tool_task_join_error)?;
}
// The abort owns the terminal outcome; await only so
// the runtime can finish process teardown.
match handle.await {
Ok(_) => {}
Err(err) if err.is_cancelled() => {}
Err(err) => return Err(Self::tool_task_join_error(err)),
}
} else {
handle.abort();
match handle.await {
Ok(result) => return result,
Err(err) if err.is_cancelled() => {}
Err(err) => return Err(Self::tool_task_join_error(err)),
}
Err(err) => Err(Self::tool_task_join_error(err)),
}
let response = Self::aborted_response(&call, secs);
notify_tool_aborted(
abort_session.as_ref(),
abort_turn.as_ref(),
call.call_id.as_str(),
&call.tool_name,
abort_source,
)
.await;
Ok(response)
}
},
}
@@ -274,6 +287,85 @@ mod tests {
impl CoreToolRuntime for ImmediateHandler {}
struct CancellationCleanupHandler {
tool_name: codex_tools::ToolName,
started: std::sync::Mutex<Option<oneshot::Sender<()>>>,
cleanup_started: std::sync::Mutex<Option<oneshot::Sender<()>>>,
allow_cleanup: Arc<Notify>,
}
#[async_trait::async_trait]
impl ToolExecutor<ToolInvocation> for CancellationCleanupHandler {
fn tool_name(&self) -> codex_tools::ToolName {
self.tool_name.clone()
}
fn spec(&self) -> codex_tools::ToolSpec {
codex_tools::ToolSpec::Function(codex_tools::ResponsesApiTool {
name: self.tool_name.name.clone(),
description: "Cancellation cleanup test tool.".to_string(),
strict: false,
defer_loading: None,
parameters: codex_tools::JsonSchema::default(),
output_schema: None,
})
}
async fn handle(
&self,
invocation: ToolInvocation,
) -> Result<Box<dyn crate::tools::context::ToolOutput>, FunctionCallError> {
let started = self
.started
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
if let Some(started) = started {
let _ = started.send(());
}
invocation.cancellation_token.cancelled().await;
let cleanup_started = self
.cleanup_started
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
if let Some(cleanup_started) = cleanup_started {
let _ = cleanup_started.send(());
}
self.allow_cleanup.notified().await;
Ok(Box::new(FunctionToolOutput::from_text(
"cleanup complete".to_string(),
Some(false),
)))
}
}
impl CoreToolRuntime for CancellationCleanupHandler {
fn waits_for_runtime_cancellation(&self) -> bool {
true
}
}
struct FinishRecorder {
records: Arc<std::sync::Mutex<Vec<ToolCallOutcome>>>,
}
impl codex_extension_api::ToolLifecycleContributor for FinishRecorder {
fn on_tool_finish<'a>(
&'a self,
input: codex_extension_api::ToolFinishInput<'a>,
) -> codex_extension_api::ToolLifecycleFuture<'a> {
let records = Arc::clone(&self.records);
let outcome = input.outcome;
Box::pin(async move {
records
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(outcome);
})
}
}
struct BlockingFinishContributor {
records: Arc<std::sync::Mutex<Vec<ToolCallOutcome>>>,
finish_started: std::sync::Mutex<Option<oneshot::Sender<()>>>,
@@ -375,4 +467,75 @@ mod tests {
Ok(())
}
#[tokio::test]
async fn cancellation_waiting_for_runtime_cleanup_emits_only_aborted_lifecycle()
-> anyhow::Result<()> {
let (mut session, turn_context) = crate::session::tests::make_session_and_context().await;
let records = Arc::new(std::sync::Mutex::new(Vec::new()));
let mut builder =
codex_extension_api::ExtensionRegistryBuilder::<crate::config::Config>::new();
builder.tool_lifecycle_contributor(Arc::new(FinishRecorder {
records: Arc::clone(&records),
}));
session.services.extensions = Arc::new(builder.build());
let session = Arc::new(session);
let turn_context = Arc::new(turn_context);
let tool_name = codex_tools::ToolName::plain("cleanup_tool");
let (started_tx, started_rx) = oneshot::channel();
let (cleanup_started_tx, cleanup_started_rx) = oneshot::channel();
let allow_cleanup = Arc::new(Notify::new());
let handler = Arc::new(CancellationCleanupHandler {
tool_name: tool_name.clone(),
started: std::sync::Mutex::new(Some(started_tx)),
cleanup_started: std::sync::Mutex::new(Some(cleanup_started_tx)),
allow_cleanup: Arc::clone(&allow_cleanup),
}) as Arc<dyn CoreToolRuntime>;
let router = Arc::new(ToolRouter::from_parts(
ToolRegistry::from_tools([handler]),
Vec::new(),
));
let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new()));
let runtime = ToolCallRuntime::new(router, session, turn_context, tracker);
let cancellation_token = CancellationToken::new();
let call = ToolCall {
tool_name,
call_id: "call-1".to_string(),
payload: ToolPayload::Function {
arguments: "{}".to_string(),
},
};
let response_task =
tokio::spawn(runtime.handle_tool_call(call, cancellation_token.clone()));
started_rx.await.expect("handler should start");
cancellation_token.cancel();
cleanup_started_rx
.await
.expect("handler should start cleanup");
tokio::time::sleep(Duration::from_millis(10)).await;
allow_cleanup.notify_one();
let response = tokio::time::timeout(Duration::from_secs(1), response_task)
.await
.expect("timed out waiting for tool response")
.expect("tool response task should join")?;
let ResponseInputItem::FunctionCallOutput { output, .. } = response else {
anyhow::bail!("cancelled tool should return function output");
};
let FunctionCallOutputBody::Text(text) = output.body else {
anyhow::bail!("cancelled tool output should be text");
};
assert!(text.contains("aborted by user"));
let actual = records
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.drain(..)
.collect::<Vec<_>>();
assert_eq!(vec![ToolCallOutcome::Aborted], actual);
Ok(())
}
}
+50 -19
View File
@@ -57,6 +57,12 @@ pub(crate) trait CoreToolRuntime: ToolExecutor<ToolInvocation> {
)
}
/// Whether cancellation should let the handler finish teardown before the
/// host returns an aborted tool response.
fn waits_for_runtime_cancellation(&self) -> bool {
false
}
fn telemetry_tags<'a>(
&'a self,
_invocation: &'a ToolInvocation,
@@ -284,6 +290,10 @@ impl CoreToolRuntime for ExposureOverride {
self.handler.matches_kind(payload)
}
fn waits_for_runtime_cancellation(&self) -> bool {
self.handler.waits_for_runtime_cancellation()
}
fn pre_tool_use_payload(&self, invocation: &ToolInvocation) -> Option<PreToolUsePayload> {
self.handler.pre_tool_use_payload(invocation)
}
@@ -381,6 +391,11 @@ impl ToolRegistry {
Some(tool.supports_parallel_tool_calls())
}
pub(crate) fn waits_for_runtime_cancellation(&self, name: &ToolName) -> Option<bool> {
let tool = self.tool(name)?;
Some(tool.waits_for_runtime_cancellation())
}
#[allow(dead_code)]
pub(crate) async fn dispatch_any(
&self,
@@ -497,10 +512,12 @@ impl ToolRegistry {
PreToolUseHookResult::Blocked(message) => {
let err = FunctionCallError::RespondToModel(message);
dispatch_trace.record_failed(&err);
if let Some(terminal_outcome_reached) = &terminal_outcome_reached {
terminal_outcome_reached.store(true, Ordering::Release);
}
notify_tool_finish(&invocation, ToolCallOutcome::Blocked).await;
notify_tool_finish_if_unclaimed(
&invocation,
terminal_outcome_reached.as_deref(),
ToolCallOutcome::Blocked,
)
.await;
return Err(err);
}
PreToolUseHookResult::Continue {
@@ -511,11 +528,9 @@ impl ToolRegistry {
}
Err(err) => {
dispatch_trace.record_failed(&err);
if let Some(terminal_outcome_reached) = &terminal_outcome_reached {
terminal_outcome_reached.store(true, Ordering::Release);
}
notify_tool_finish(
notify_tool_finish_if_unclaimed(
&invocation,
terminal_outcome_reached.as_deref(),
ToolCallOutcome::Failed {
handler_executed: false,
},
@@ -638,18 +653,21 @@ impl ToolRegistry {
handler_executed: true,
},
};
if let Some(terminal_outcome_reached) = &terminal_outcome_reached {
terminal_outcome_reached.store(true, Ordering::Release);
}
notify_tool_finish(&invocation, lifecycle_outcome).await;
let finished = notify_tool_finish_if_unclaimed(
&invocation,
terminal_outcome_reached.as_deref(),
lifecycle_outcome,
)
.await;
if let Err(err) = invocation
.session
.goal_runtime_apply(GoalRuntimeEvent::ToolCompleted {
turn_context: invocation.turn.as_ref(),
tool_name: tool_name.name.as_str(),
})
.await
if finished
&& let Err(err) = invocation
.session
.goal_runtime_apply(GoalRuntimeEvent::ToolCompleted {
turn_context: invocation.turn.as_ref(),
tool_name: tool_name.name.as_str(),
})
.await
{
warn!("failed to account thread goal progress after tool call: {err}");
}
@@ -676,6 +694,19 @@ impl ToolRegistry {
}
}
async fn notify_tool_finish_if_unclaimed(
invocation: &ToolInvocation,
terminal_outcome_reached: Option<&AtomicBool>,
outcome: ToolCallOutcome,
) -> bool {
if terminal_outcome_reached.is_some_and(|reached| reached.swap(true, Ordering::AcqRel)) {
return false;
}
notify_tool_finish(invocation, outcome).await;
true
}
async fn handle_any_tool(
tool: &dyn CoreToolRuntime,
invocation: ToolInvocation,
+6
View File
@@ -86,6 +86,12 @@ impl ToolRouter {
.unwrap_or(false)
}
pub fn tool_waits_for_runtime_cancellation(&self, call: &ToolCall) -> bool {
self.registry
.waits_for_runtime_cancellation(&call.tool_name)
.unwrap_or(false)
}
#[instrument(level = "trace", skip_all, err)]
pub fn build_tool_call(item: ResponseItem) -> Result<Option<ToolCall>, FunctionCallError> {
match item {
@@ -44,6 +44,7 @@ use codex_shell_command::powershell::prefix_powershell_script_with_utf8;
use codex_utils_absolute_path::AbsolutePathBuf;
use futures::future::BoxFuture;
use std::collections::HashMap;
use tokio_util::sync::CancellationToken;
#[derive(Clone, Debug)]
pub struct ShellRequest {
@@ -52,6 +53,7 @@ pub struct ShellRequest {
pub hook_command: String,
pub cwd: AbsolutePathBuf,
pub timeout_ms: Option<u64>,
pub cancellation_token: CancellationToken,
pub env: HashMap<String, String>,
pub explicit_env_overrides: HashMap<String, String>,
pub network: Option<NetworkProxy>,
@@ -265,6 +267,7 @@ impl ToolRuntime<ShellRequest, ExecToolCallOutput> for ShellRuntime {
let command =
build_sandbox_command(&command, &req.cwd, &env, req.additional_permissions.clone())?;
let mut expiration: crate::exec::ExecExpiration = req.timeout_ms.into();
expiration = expiration.with_cancellation(req.cancellation_token.clone());
if let Some(cancellation) = attempt.network_denial_cancellation_token.clone() {
expiration = expiration.with_cancellation(cancellation);
}