From ecae41274091252f454eb5ed0507036ed4e6fc19 Mon Sep 17 00:00:00 2001 From: rreichel3-oai Date: Thu, 4 Jun 2026 20:58:14 -0400 Subject: [PATCH] [codex] Emit sandbox outcome telemetry event (#25955) ## Summary Adds a dedicated `codex.sandbox_outcome` telemetry event so we can query sandbox edge outcomes without threading sandbox metadata through tool-result output types. This is meant to make sandbox failures and approved escalation retries visible in OTEL while keeping the existing `codex.tool_result` event shape focused on tool completion data. ## What changed - Adds `SessionTelemetry::sandbox_outcome(...)`, which emits `codex.sandbox_outcome` as both a log and trace event. - Records the tool name, call id, sandbox outcome, initial attempt duration, and escalated attempt duration when a retry runs. - Emits `denied` when the sandbox blocks execution and no retry is run. - Emits `timed_out` and `signal` when those sandbox errors surface from tool execution. - Emits `escalated` when the initial sandboxed attempt fails and the approved unsandboxed retry succeeds. - Adds OTEL coverage for the new event payload, including timing fields. ## Validation - `RUST_MIN_STACK=8388608 just test -p codex-core sandbox_outcome_event_records_outcome handle_sandbox_error_user_approves_retry_records_tool_decision` - `just test -p codex-otel otel_export_routing_policy_routes_tool_result_log_and_trace_events runtime_metrics_summary_collects_tool_api_and_streaming_metrics` - `just fix -p codex-core` - `just fix -p codex-otel` --- codex-rs/core/src/tools/orchestrator.rs | 86 +++++++++++++++++-- codex-rs/core/tests/suite/otel.rs | 69 +++++++++++++++ codex-rs/otel/src/events/session_telemetry.rs | 31 +++++++ 3 files changed, 181 insertions(+), 5 deletions(-) diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index 235f8c4e9..77cbffde3 100644 --- a/codex-rs/core/src/tools/orchestrator.rs +++ b/codex-rs/core/src/tools/orchestrator.rs @@ -39,6 +39,7 @@ use codex_protocol::protocol::NetworkPolicyRuleAction; use codex_protocol::protocol::ReviewDecision; use codex_sandboxing::SandboxManager; use codex_sandboxing::SandboxType; +use std::time::Instant; pub(crate) struct ToolOrchestrator { sandbox: SandboxManager, @@ -256,6 +257,7 @@ impl ToolOrchestrator { network_denial_cancellation_token: None, }; + let initial_attempt_start = Instant::now(); let (first_result, first_deferred_network_approval) = Self::run_attempt( tool, req, @@ -264,6 +266,7 @@ impl ToolOrchestrator { managed_network_active, ) .await; + let initial_duration = initial_attempt_start.elapsed(); match first_result { Ok(out) => { // We have a successful initial result @@ -284,12 +287,26 @@ impl ToolOrchestrator { None }; if network_policy_decision.is_some() && network_approval_context.is_none() { + otel.sandbox_outcome( + &otel_tn, + otel_ci, + "denied", + initial_duration, + /*escalated_duration*/ None, + ); return Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { output, network_policy_decision, }))); } if !tool.escalate_on_failure() { + otel.sandbox_outcome( + &otel_tn, + otel_ci, + "denied", + initial_duration, + /*escalated_duration*/ None, + ); return Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { output, network_policy_decision, @@ -312,6 +329,13 @@ impl ToolOrchestrator { ExecApprovalRequirement::NeedsApproval { .. } ); if !allow_on_request_network_prompt { + otel.sandbox_outcome( + &otel_tn, + otel_ci, + "denied", + initial_duration, + /*escalated_duration*/ None, + ); return Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { output, network_policy_decision, @@ -319,6 +343,13 @@ impl ToolOrchestrator { } } if !unsandboxed_allowed && network_approval_context.is_none() { + otel.sandbox_outcome( + &otel_tn, + otel_ci, + "denied", + initial_duration, + /*escalated_duration*/ None, + ); return Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { output, network_policy_decision, @@ -400,15 +431,51 @@ impl ToolOrchestrator { }; // Second attempt. + let escalated_attempt_start = Instant::now(); let (retry_result, retry_deferred_network_approval) = Self::run_attempt(tool, req, tool_ctx, &retry_attempt, managed_network_active) .await; - retry_result.map(|output| OrchestratorRunResult { - output, - deferred_network_approval: retry_deferred_network_approval, - }) + let escalated_duration = escalated_attempt_start.elapsed(); + match retry_result { + Ok(output) => { + otel.sandbox_outcome( + &otel_tn, + otel_ci, + "escalated", + initial_duration, + Some(escalated_duration), + ); + Ok(OrchestratorRunResult { + output, + deferred_network_approval: retry_deferred_network_approval, + }) + } + Err(err) => { + if let Some(outcome) = sandbox_outcome_from_tool_error(&err) { + otel.sandbox_outcome( + &otel_tn, + otel_ci, + outcome, + initial_duration, + Some(escalated_duration), + ); + } + Err(err) + } + } + } + Err(err) => { + if let Some(outcome) = sandbox_outcome_from_tool_error(&err) { + otel.sandbox_outcome( + &otel_tn, + otel_ci, + outcome, + initial_duration, + /*escalated_duration*/ None, + ); + } + Err(err) } - Err(err) => Err(err), } } @@ -509,6 +576,15 @@ impl ToolOrchestrator { } } +fn sandbox_outcome_from_tool_error(err: &ToolError) -> Option<&'static str> { + match err { + ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { .. })) => Some("denied"), + ToolError::Codex(CodexErr::Sandbox(SandboxErr::Timeout { .. })) => Some("timed_out"), + ToolError::Codex(CodexErr::Sandbox(SandboxErr::Signal(_))) => Some("signal"), + ToolError::Rejected(_) | ToolError::Codex(_) => None, + } +} + fn build_denial_reason_from_output(_output: &ExecToolCallOutput) -> String { // Keep approval reason terse and stable for UX/tests, but accept the // output so we can evolve heuristics later without touching call sites. diff --git a/codex-rs/core/tests/suite/otel.rs b/codex-rs/core/tests/suite/otel.rs index 7f5fa260c..d6c7577a7 100644 --- a/codex-rs/core/tests/suite/otel.rs +++ b/codex-rs/core/tests/suite/otel.rs @@ -1,11 +1,15 @@ use codex_core::config::Constrained; use codex_features::Feature; +use codex_otel::SessionTelemetry; +use codex_otel::TelemetryAuthMode; +use codex_protocol::ThreadId; use codex_protocol::models::PermissionProfile; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; use codex_protocol::protocol::ReviewDecision; +use codex_protocol::protocol::SessionSource; use codex_protocol::user_input::UserInput; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; @@ -27,6 +31,7 @@ use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use std::sync::Mutex; +use std::time::Duration; use tracing::Level; use tracing_test::traced_test; @@ -1145,6 +1150,70 @@ fn tool_decision_assertion<'a>( } } +fn sandbox_outcome_assertion<'a>( + call_id: &'a str, + expected_outcome: &'a str, +) -> impl Fn(&[&str]) -> Result<(), String> + 'a { + let call_id = call_id.to_string(); + let expected_outcome = expected_outcome.to_string(); + + move |lines: &[&str]| { + let line = lines + .iter() + .find(|line| { + line.contains("codex.sandbox_outcome") + && line.contains(&format!("call_id={call_id}")) + }) + .ok_or_else(|| format!("missing codex.sandbox_outcome event for {call_id}"))?; + + let lower = line.to_lowercase(); + if !lower.contains("tool_name=shell_command") { + return Err("missing tool_name for shell_command".to_string()); + } + if !lower.contains(&format!("outcome={expected_outcome}")) { + return Err(format!("unexpected sandbox outcome for {call_id}")); + } + if !lower.contains("initial_duration_ms=12") { + return Err("missing initial_duration_ms field".to_string()); + } + if !lower.contains("escalated_duration_ms=34") { + return Err("missing escalated_duration_ms field".to_string()); + } + + Ok(()) + } +} + +#[test] +#[traced_test] +fn sandbox_outcome_event_records_outcome() { + let telemetry = SessionTelemetry::new( + ThreadId::new(), + "gpt-5.5", + "gpt-5.5", + /*account_id*/ None, + /*account_email*/ None, + Some(TelemetryAuthMode::ApiKey), + "Codex_Desktop".to_string(), + /*log_user_prompts*/ false, + "tty".to_string(), + SessionSource::Cli, + ); + + telemetry.sandbox_outcome( + "shell_command", + "sandbox-outcome-call", + "escalated", + Duration::from_millis(/*millis*/ 12), + Some(Duration::from_millis(/*millis*/ 34)), + ); + + logs_assert(sandbox_outcome_assertion( + "sandbox-outcome-call", + "escalated", + )); +} + #[tokio::test] #[traced_test] async fn handle_shell_command_autoapprove_from_config_records_tool_decision() { diff --git a/codex-rs/otel/src/events/session_telemetry.rs b/codex-rs/otel/src/events/session_telemetry.rs index f4534a079..0003cb04a 100644 --- a/codex-rs/otel/src/events/session_telemetry.rs +++ b/codex-rs/otel/src/events/session_telemetry.rs @@ -998,6 +998,37 @@ impl SessionTelemetry { ); } + pub fn sandbox_outcome( + &self, + tool_name: &str, + call_id: &str, + outcome: &str, + initial_duration: Duration, + escalated_duration: Option, + ) { + let initial_duration_ms = initial_duration.as_millis().min(i64::MAX as u128) as i64; + let escalated_duration_ms = + escalated_duration.map(|duration| duration.as_millis().min(i64::MAX as u128) as i64); + log_event!( + self, + event.name = "codex.sandbox_outcome", + tool_name = %tool_name, + call_id = %call_id, + outcome = %outcome, + initial_duration_ms = initial_duration_ms, + escalated_duration_ms = escalated_duration_ms, + ); + trace_event!( + self, + event.name = "codex.sandbox_outcome", + tool_name = %tool_name, + call_id = %call_id, + outcome = %outcome, + initial_duration_ms = initial_duration_ms, + escalated_duration_ms = escalated_duration_ms, + ); + } + #[allow(clippy::too_many_arguments)] pub async fn log_tool_result_with_tags( &self,