[codex] Emit sandbox outcome telemetry event (#25955)

## Summary

Adds a dedicated `codex.sandbox_outcome` telemetry event so we can query
sandbox edge outcomes without threading sandbox metadata through
tool-result output types.

This is meant to make sandbox failures and approved escalation retries
visible in OTEL while keeping the existing `codex.tool_result` event
shape focused on tool completion data.

## What changed

- Adds `SessionTelemetry::sandbox_outcome(...)`, which emits
`codex.sandbox_outcome` as both a log and trace event.
- Records the tool name, call id, sandbox outcome, initial attempt
duration, and escalated attempt duration when a retry runs.
- Emits `denied` when the sandbox blocks execution and no retry is run.
- Emits `timed_out` and `signal` when those sandbox errors surface from
tool execution.
- Emits `escalated` when the initial sandboxed attempt fails and the
approved unsandboxed retry succeeds.
- Adds OTEL coverage for the new event payload, including timing fields.

## Validation

- `RUST_MIN_STACK=8388608 just test -p codex-core
sandbox_outcome_event_records_outcome
handle_sandbox_error_user_approves_retry_records_tool_decision`
- `just test -p codex-otel
otel_export_routing_policy_routes_tool_result_log_and_trace_events
runtime_metrics_summary_collects_tool_api_and_streaming_metrics`
- `just fix -p codex-core`
- `just fix -p codex-otel`
This commit is contained in:
rreichel3-oai
2026-06-04 20:58:14 -04:00
committed by GitHub
parent 4be1a168fc
commit ecae412740
3 changed files with 181 additions and 5 deletions
+69
View File
@@ -1,11 +1,15 @@
use codex_core::config::Constrained;
use codex_features::Feature;
use codex_otel::SessionTelemetry;
use codex_otel::TelemetryAuthMode;
use codex_protocol::ThreadId;
use codex_protocol::models::PermissionProfile;
use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::Op;
use codex_protocol::protocol::ReviewDecision;
use codex_protocol::protocol::SessionSource;
use codex_protocol::user_input::UserInput;
use core_test_support::responses::ev_assistant_message;
use core_test_support::responses::ev_completed;
@@ -27,6 +31,7 @@ use core_test_support::test_codex::TestCodex;
use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
use std::sync::Mutex;
use std::time::Duration;
use tracing::Level;
use tracing_test::traced_test;
@@ -1145,6 +1150,70 @@ fn tool_decision_assertion<'a>(
}
}
fn sandbox_outcome_assertion<'a>(
call_id: &'a str,
expected_outcome: &'a str,
) -> impl Fn(&[&str]) -> Result<(), String> + 'a {
let call_id = call_id.to_string();
let expected_outcome = expected_outcome.to_string();
move |lines: &[&str]| {
let line = lines
.iter()
.find(|line| {
line.contains("codex.sandbox_outcome")
&& line.contains(&format!("call_id={call_id}"))
})
.ok_or_else(|| format!("missing codex.sandbox_outcome event for {call_id}"))?;
let lower = line.to_lowercase();
if !lower.contains("tool_name=shell_command") {
return Err("missing tool_name for shell_command".to_string());
}
if !lower.contains(&format!("outcome={expected_outcome}")) {
return Err(format!("unexpected sandbox outcome for {call_id}"));
}
if !lower.contains("initial_duration_ms=12") {
return Err("missing initial_duration_ms field".to_string());
}
if !lower.contains("escalated_duration_ms=34") {
return Err("missing escalated_duration_ms field".to_string());
}
Ok(())
}
}
#[test]
#[traced_test]
fn sandbox_outcome_event_records_outcome() {
let telemetry = SessionTelemetry::new(
ThreadId::new(),
"gpt-5.5",
"gpt-5.5",
/*account_id*/ None,
/*account_email*/ None,
Some(TelemetryAuthMode::ApiKey),
"Codex_Desktop".to_string(),
/*log_user_prompts*/ false,
"tty".to_string(),
SessionSource::Cli,
);
telemetry.sandbox_outcome(
"shell_command",
"sandbox-outcome-call",
"escalated",
Duration::from_millis(/*millis*/ 12),
Some(Duration::from_millis(/*millis*/ 34)),
);
logs_assert(sandbox_outcome_assertion(
"sandbox-outcome-call",
"escalated",
));
}
#[tokio::test]
#[traced_test]
async fn handle_shell_command_autoapprove_from_config_records_tool_decision() {