mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
[codex] Emit sandbox outcome telemetry event (#25955)
## Summary Adds a dedicated `codex.sandbox_outcome` telemetry event so we can query sandbox edge outcomes without threading sandbox metadata through tool-result output types. This is meant to make sandbox failures and approved escalation retries visible in OTEL while keeping the existing `codex.tool_result` event shape focused on tool completion data. ## What changed - Adds `SessionTelemetry::sandbox_outcome(...)`, which emits `codex.sandbox_outcome` as both a log and trace event. - Records the tool name, call id, sandbox outcome, initial attempt duration, and escalated attempt duration when a retry runs. - Emits `denied` when the sandbox blocks execution and no retry is run. - Emits `timed_out` and `signal` when those sandbox errors surface from tool execution. - Emits `escalated` when the initial sandboxed attempt fails and the approved unsandboxed retry succeeds. - Adds OTEL coverage for the new event payload, including timing fields. ## Validation - `RUST_MIN_STACK=8388608 just test -p codex-core sandbox_outcome_event_records_outcome handle_sandbox_error_user_approves_retry_records_tool_decision` - `just test -p codex-otel otel_export_routing_policy_routes_tool_result_log_and_trace_events runtime_metrics_summary_collects_tool_api_and_streaming_metrics` - `just fix -p codex-core` - `just fix -p codex-otel`
This commit is contained in:
@@ -1,11 +1,15 @@
|
||||
use codex_core::config::Constrained;
|
||||
use codex_features::Feature;
|
||||
use codex_otel::SessionTelemetry;
|
||||
use codex_otel::TelemetryAuthMode;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::openai_models::ReasoningEffort;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::Op;
|
||||
use codex_protocol::protocol::ReviewDecision;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use core_test_support::responses::ev_assistant_message;
|
||||
use core_test_support::responses::ev_completed;
|
||||
@@ -27,6 +31,7 @@ use core_test_support::test_codex::TestCodex;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use core_test_support::wait_for_event;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
use tracing::Level;
|
||||
use tracing_test::traced_test;
|
||||
|
||||
@@ -1145,6 +1150,70 @@ fn tool_decision_assertion<'a>(
|
||||
}
|
||||
}
|
||||
|
||||
fn sandbox_outcome_assertion<'a>(
|
||||
call_id: &'a str,
|
||||
expected_outcome: &'a str,
|
||||
) -> impl Fn(&[&str]) -> Result<(), String> + 'a {
|
||||
let call_id = call_id.to_string();
|
||||
let expected_outcome = expected_outcome.to_string();
|
||||
|
||||
move |lines: &[&str]| {
|
||||
let line = lines
|
||||
.iter()
|
||||
.find(|line| {
|
||||
line.contains("codex.sandbox_outcome")
|
||||
&& line.contains(&format!("call_id={call_id}"))
|
||||
})
|
||||
.ok_or_else(|| format!("missing codex.sandbox_outcome event for {call_id}"))?;
|
||||
|
||||
let lower = line.to_lowercase();
|
||||
if !lower.contains("tool_name=shell_command") {
|
||||
return Err("missing tool_name for shell_command".to_string());
|
||||
}
|
||||
if !lower.contains(&format!("outcome={expected_outcome}")) {
|
||||
return Err(format!("unexpected sandbox outcome for {call_id}"));
|
||||
}
|
||||
if !lower.contains("initial_duration_ms=12") {
|
||||
return Err("missing initial_duration_ms field".to_string());
|
||||
}
|
||||
if !lower.contains("escalated_duration_ms=34") {
|
||||
return Err("missing escalated_duration_ms field".to_string());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[traced_test]
|
||||
fn sandbox_outcome_event_records_outcome() {
|
||||
let telemetry = SessionTelemetry::new(
|
||||
ThreadId::new(),
|
||||
"gpt-5.5",
|
||||
"gpt-5.5",
|
||||
/*account_id*/ None,
|
||||
/*account_email*/ None,
|
||||
Some(TelemetryAuthMode::ApiKey),
|
||||
"Codex_Desktop".to_string(),
|
||||
/*log_user_prompts*/ false,
|
||||
"tty".to_string(),
|
||||
SessionSource::Cli,
|
||||
);
|
||||
|
||||
telemetry.sandbox_outcome(
|
||||
"shell_command",
|
||||
"sandbox-outcome-call",
|
||||
"escalated",
|
||||
Duration::from_millis(/*millis*/ 12),
|
||||
Some(Duration::from_millis(/*millis*/ 34)),
|
||||
);
|
||||
|
||||
logs_assert(sandbox_outcome_assertion(
|
||||
"sandbox-outcome-call",
|
||||
"escalated",
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[traced_test]
|
||||
async fn handle_shell_command_autoapprove_from_config_records_tool_decision() {
|
||||
|
||||
Reference in New Issue
Block a user