From 8e4b92d294c087423215a83b0e9c8282eedc9d12 Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Tue, 2 Jun 2026 11:40:35 -0700 Subject: [PATCH] [codex-analytics] Track CodexErr details in turn analytics (#25707) ## Summary - add analytics-only `CodexErr` telemetry to `codex_turn_event` while leaving existing `turn_error` unchanged - record terminal `CodexErr` facts from core immediately before the existing turn error event is sent - emit source-truth `codex_error_*` fields for downstream analytics, including the raw `CodexErr::InvalidRequest(String)` message as `codex_error_subreason` ## Validation - `just test -p codex-analytics` - attempted `just test -p codex-core`, but the local run timed out across unrelated integration suites in this environment and is not being used as validation --- .../analytics/src/analytics_client_tests.rs | 33 ++++ codex-rs/analytics/src/client.rs | 7 + codex-rs/analytics/src/events.rs | 4 + codex-rs/analytics/src/facts.rs | 146 ++++++++++++++++++ codex-rs/analytics/src/lib.rs | 1 + codex-rs/analytics/src/reducer.rs | 39 +++++ codex-rs/core/src/compact.rs | 2 + codex-rs/core/src/compact_remote.rs | 1 + codex-rs/core/src/compact_remote_v2.rs | 1 + codex-rs/core/src/session/mod.rs | 12 ++ codex-rs/core/src/session/turn.rs | 4 +- 11 files changed, 249 insertions(+), 1 deletion(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index b3f6ba22c..61ab5f6b2 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -62,6 +62,7 @@ use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; use crate::facts::ThreadInitializationMode; use crate::facts::TrackEventsContext; +use crate::facts::TurnCodexErrorFact; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; use crate::facts::TurnSteerRequestError; @@ -132,6 +133,7 @@ use codex_plugin::PluginTelemetryMetadata; use codex_protocol::approvals::NetworkApprovalProtocol; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::config_types::ModeKind; +use codex_protocol::error::CodexErr; use codex_protocol::models::NetworkPermissions as CoreNetworkPermissions; use codex_protocol::models::PermissionProfile as CorePermissionProfile; use codex_protocol::protocol::AskForApproval; @@ -3256,6 +3258,9 @@ fn turn_event_serializes_expected_shape() { is_first_turn: true, status: Some(TurnStatus::Completed), turn_error: None, + codex_error_kind: None, + codex_error_subreason: None, + codex_error_http_status_code: None, steer_count: Some(0), total_tool_call_count: None, shell_command_count: None, @@ -3318,6 +3323,9 @@ fn turn_event_serializes_expected_shape() { "is_first_turn": true, "status": "completed", "turn_error": null, + "codex_error_kind": null, + "codex_error_subreason": null, + "codex_error_http_status_code": null, "steer_count": 0, "total_tool_call_count": null, "shell_command_count": null, @@ -3983,6 +3991,18 @@ async fn turn_lifecycle_emits_failed_turn_event() { /*include_token_usage*/ false, ) .await; + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnCodexError(Box::new( + TurnCodexErrorFact::from_codex_err( + "thread-2".to_string(), + "turn-2".to_string(), + &CodexErr::InvalidRequest("unknown turn environment id `env-2`".to_string()), + ), + ))), + &mut out, + ) + .await; reducer .ingest( AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( @@ -3999,6 +4019,18 @@ async fn turn_lifecycle_emits_failed_turn_event() { let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); assert_eq!(payload["event_params"]["status"], json!("failed")); assert_eq!(payload["event_params"]["turn_error"], json!("badRequest")); + assert_eq!( + payload["event_params"]["codex_error_kind"], + json!("invalid_request") + ); + assert_eq!( + payload["event_params"]["codex_error_subreason"], + json!("unknown turn environment id `env-2`") + ); + assert_eq!( + payload["event_params"]["codex_error_http_status_code"], + json!(null) + ); } #[tokio::test] @@ -4031,6 +4063,7 @@ async fn turn_lifecycle_emits_interrupted_turn_event_without_error() { let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); assert_eq!(payload["event_params"]["status"], json!("interrupted")); assert_eq!(payload["event_params"]["turn_error"], json!(null)); + assert_eq!(payload["event_params"]["codex_error_kind"], json!(null)); } #[tokio::test] diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index fbcfa32dc..bd0726b28 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -18,6 +18,7 @@ use crate::facts::SkillInvocation; use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; +use crate::facts::TurnCodexErrorFact; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnTokenUsageFact; use crate::reducer::AnalyticsReducer; @@ -256,6 +257,12 @@ impl AnalyticsEventsClient { ))); } + pub fn track_turn_codex_error(&self, fact: TurnCodexErrorFact) { + self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::TurnCodexError( + Box::new(fact), + ))); + } + pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput { diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index fd52fefc1..d2e7d8054 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -3,6 +3,7 @@ use std::time::Instant; use crate::facts::AcceptedLineFingerprint; use crate::facts::AppInvocation; use crate::facts::CodexCompactionEvent; +use crate::facts::CodexErrKind; use crate::facts::CompactionImplementation; use crate::facts::CompactionPhase; use crate::facts::CompactionReason; @@ -797,6 +798,9 @@ pub(crate) struct CodexTurnEventParams { pub(crate) is_first_turn: bool, pub(crate) status: Option, pub(crate) turn_error: Option, + pub(crate) codex_error_kind: Option, + pub(crate) codex_error_subreason: Option, + pub(crate) codex_error_http_status_code: Option, pub(crate) steer_count: Option, pub(crate) total_tool_call_count: Option, pub(crate) shell_command_count: Option, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 9a8527671..4d298ffdb 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -15,6 +15,7 @@ use codex_protocol::config_types::ModeKind; use codex_protocol::config_types::Personality; use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::ServiceTier; +use codex_protocol::error::CodexErr; use codex_protocol::models::PermissionProfile; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::AskForApproval; @@ -29,6 +30,9 @@ use codex_protocol::request_permissions::RequestPermissionsResponse; use serde::Serialize; use std::path::PathBuf; +const INVALID_REQUEST_SUBREASON_MAX_BYTES: usize = 512; +const INVALID_REQUEST_SUBREASON_TRUNCATION_SUFFIX: &str = "..."; + #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct AcceptedLineFingerprint { pub path_hash: String, @@ -99,6 +103,147 @@ pub struct TurnTokenUsageFact { pub token_usage: TokenUsage, } +#[derive(Clone)] +pub struct TurnCodexErrorFact { + pub(crate) turn_id: String, + pub(crate) thread_id: String, + pub(crate) error: TurnCodexError, +} + +impl TurnCodexErrorFact { + pub fn from_codex_err(thread_id: String, turn_id: String, error: &CodexErr) -> Self { + Self { + turn_id, + thread_id, + error: TurnCodexError::from_codex_err(error), + } + } +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum CodexErrKind { + TurnAborted, + Stream, + ContextWindowExceeded, + ThreadNotFound, + AgentLimitReached, + SessionConfiguredNotFirstEvent, + Timeout, + RequestTimeout, + Spawn, + Interrupted, + UnexpectedStatus, + InvalidRequest, + InvalidImageRequest, + UsageLimitReached, + ServerOverloaded, + CyberPolicy, + ResponseStreamFailed, + ConnectionFailed, + QuotaExceeded, + UsageNotIncluded, + InternalServerError, + RetryLimit, + InternalAgentDied, + Sandbox, + LandlockSandboxExecutableNotProvided, + UnsupportedOperation, + RefreshTokenFailed, + Fatal, + Io, + Json, + #[cfg(target_os = "linux")] + LandlockRuleset, + #[cfg(target_os = "linux")] + LandlockPathFd, + TokioJoin, + EnvVar, +} + +#[derive(Clone)] +pub(crate) struct TurnCodexError { + pub(crate) kind: CodexErrKind, + pub(crate) subreason: Option, + pub(crate) http_status_code: Option, +} + +impl TurnCodexError { + fn from_codex_err(error: &CodexErr) -> Self { + Self { + kind: error.into(), + subreason: match error { + CodexErr::InvalidRequest(message) => { + // InvalidRequest can contain raw provider response bodies, so bound the + // analytics copy without changing the source CodexErr. + let subreason = if message.len() <= INVALID_REQUEST_SUBREASON_MAX_BYTES { + message.clone() + } else { + let truncated_len = message.floor_char_boundary( + INVALID_REQUEST_SUBREASON_MAX_BYTES + .saturating_sub(INVALID_REQUEST_SUBREASON_TRUNCATION_SUFFIX.len()), + ); + format!( + "{}{INVALID_REQUEST_SUBREASON_TRUNCATION_SUFFIX}", + &message[..truncated_len] + ) + }; + Some(subreason) + } + _ => None, + }, + http_status_code: error.http_status_code_value(), + } + } +} + +impl From<&CodexErr> for CodexErrKind { + fn from(error: &CodexErr) -> Self { + match error { + CodexErr::TurnAborted => CodexErrKind::TurnAborted, + CodexErr::Stream(..) => CodexErrKind::Stream, + CodexErr::ContextWindowExceeded => CodexErrKind::ContextWindowExceeded, + CodexErr::ThreadNotFound(_) => CodexErrKind::ThreadNotFound, + CodexErr::AgentLimitReached { .. } => CodexErrKind::AgentLimitReached, + CodexErr::SessionConfiguredNotFirstEvent => { + CodexErrKind::SessionConfiguredNotFirstEvent + } + CodexErr::Timeout => CodexErrKind::Timeout, + CodexErr::RequestTimeout => CodexErrKind::RequestTimeout, + CodexErr::Spawn => CodexErrKind::Spawn, + CodexErr::Interrupted => CodexErrKind::Interrupted, + CodexErr::UnexpectedStatus(_) => CodexErrKind::UnexpectedStatus, + CodexErr::InvalidRequest(_) => CodexErrKind::InvalidRequest, + CodexErr::InvalidImageRequest() => CodexErrKind::InvalidImageRequest, + CodexErr::UsageLimitReached(_) => CodexErrKind::UsageLimitReached, + CodexErr::ServerOverloaded => CodexErrKind::ServerOverloaded, + CodexErr::CyberPolicy { .. } => CodexErrKind::CyberPolicy, + CodexErr::ResponseStreamFailed(_) => CodexErrKind::ResponseStreamFailed, + CodexErr::ConnectionFailed(_) => CodexErrKind::ConnectionFailed, + CodexErr::QuotaExceeded => CodexErrKind::QuotaExceeded, + CodexErr::UsageNotIncluded => CodexErrKind::UsageNotIncluded, + CodexErr::InternalServerError => CodexErrKind::InternalServerError, + CodexErr::RetryLimit(_) => CodexErrKind::RetryLimit, + CodexErr::InternalAgentDied => CodexErrKind::InternalAgentDied, + CodexErr::Sandbox(_) => CodexErrKind::Sandbox, + CodexErr::LandlockSandboxExecutableNotProvided => { + CodexErrKind::LandlockSandboxExecutableNotProvided + } + CodexErr::UnsupportedOperation(_) => CodexErrKind::UnsupportedOperation, + CodexErr::RefreshTokenFailed(_) => CodexErrKind::RefreshTokenFailed, + CodexErr::Fatal(_) => CodexErrKind::Fatal, + CodexErr::Io(_) => CodexErrKind::Io, + CodexErr::Json(_) => CodexErrKind::Json, + #[cfg(target_os = "linux")] + CodexErr::LandlockRuleset(_) => CodexErrKind::LandlockRuleset, + #[cfg(target_os = "linux")] + CodexErr::LandlockPathFd(_) => CodexErrKind::LandlockPathFd, + CodexErr::TokioJoin(_) => CodexErrKind::TokioJoin, + CodexErr::EnvVar(_) => CodexErrKind::EnvVar, + } + } +} + #[derive(Clone, Copy, Debug, Serialize)] #[serde(rename_all = "snake_case")] pub enum TurnStatus { @@ -329,6 +474,7 @@ pub(crate) enum CustomAnalyticsFact { GuardianReview(Box), TurnResolvedConfig(Box), TurnTokenUsage(Box), + TurnCodexError(Box), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), AppUsed(AppUsedInput), diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index a33ca7b9e..c227f4daf 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -38,6 +38,7 @@ pub use facts::SkillInvocation; pub use facts::SubAgentThreadStartedInput; pub use facts::ThreadInitializationMode; pub use facts::TrackEventsContext; +pub use facts::TurnCodexErrorFact; pub use facts::TurnResolvedConfigFact; pub use facts::TurnStatus; pub use facts::TurnSteerRejectionReason; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index f20639347..46f69c59f 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -70,6 +70,8 @@ use crate::facts::PluginUsedInput; use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; use crate::facts::ThreadInitializationMode; +use crate::facts::TurnCodexError; +use crate::facts::TurnCodexErrorFact; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; use crate::facts::TurnSteerRejectionReason; @@ -322,6 +324,7 @@ struct TurnState { started_at: Option, token_usage: Option, completed: Option, + codex_error: Option, latest_diff: Option, steer_count: usize, tool_counts: TurnToolCounts, @@ -461,6 +464,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::TurnTokenUsage(input) => { self.ingest_turn_token_usage(*input, out).await; } + CustomAnalyticsFact::TurnCodexError(input) => { + self.ingest_turn_codex_error(*input); + } CustomAnalyticsFact::SkillInvoked(input) => { self.ingest_skill_invoked(input, out).await; } @@ -606,6 +612,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + codex_error: None, latest_diff: None, steer_count: 0, tool_counts: TurnToolCounts::default(), @@ -630,6 +637,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + codex_error: None, latest_diff: None, steer_count: 0, tool_counts: TurnToolCounts::default(), @@ -639,6 +647,29 @@ impl AnalyticsReducer { self.maybe_emit_turn_event(&turn_id, out).await; } + fn ingest_turn_codex_error(&mut self, input: TurnCodexErrorFact) { + let TurnCodexErrorFact { + turn_id, + thread_id, + error, + } = input; + let turn_state = self.turns.entry(turn_id).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + token_usage: None, + completed: None, + codex_error: None, + latest_diff: None, + steer_count: 0, + tool_counts: TurnToolCounts::default(), + }); + turn_state.thread_id.get_or_insert(thread_id); + turn_state.codex_error = Some(error); + } + async fn ingest_skill_invoked( &mut self, input: SkillInvokedInput, @@ -795,6 +826,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + codex_error: None, latest_diff: None, steer_count: 0, tool_counts: TurnToolCounts::default(), @@ -1154,6 +1186,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + codex_error: None, latest_diff: None, steer_count: 0, tool_counts: TurnToolCounts::default(), @@ -1175,6 +1208,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + codex_error: None, latest_diff: None, steer_count: 0, tool_counts: TurnToolCounts::default(), @@ -1194,6 +1228,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + codex_error: None, latest_diff: None, steer_count: 0, tool_counts: TurnToolCounts::default(), @@ -2451,6 +2486,7 @@ fn codex_turn_event_params( is_first_turn, } = resolved_config; let token_usage = turn_state.token_usage.clone(); + let codex_error = turn_state.codex_error.as_ref(); CodexTurnEventParams { thread_id, session_id: thread_metadata.session_id.clone(), @@ -2483,6 +2519,9 @@ fn codex_turn_event_params( is_first_turn, status: completed.status, turn_error: completed.turn_error, + codex_error_kind: codex_error.map(|error| error.kind), + codex_error_subreason: codex_error.and_then(|error| error.subreason.clone()), + codex_error_http_status_code: codex_error.and_then(|error| error.http_status_code), steer_count: Some(turn_state.steer_count), total_tool_call_count: Some(turn_state.tool_counts.total), shell_command_count: Some(turn_state.tool_counts.shell_command), diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 11386e6bc..a002ce20d 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -240,6 +240,7 @@ async fn run_compact_task_inner_impl( continue; } sess.set_total_tokens_full(turn_context.as_ref()).await; + sess.track_turn_codex_error(turn_context.as_ref(), &e); let event = EventMsg::Error(e.to_error_event(/*message_prefix*/ None)); sess.send_event(&turn_context, event).await; return Err(e); @@ -257,6 +258,7 @@ async fn run_compact_task_inner_impl( tokio::time::sleep(delay).await; continue; } else { + sess.track_turn_codex_error(turn_context.as_ref(), &e); let event = EventMsg::Error(e.to_error_event(/*message_prefix*/ None)); sess.send_event(&turn_context, event).await; return Err(e); diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index b4456dc71..c2ef417b9 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -138,6 +138,7 @@ async fn run_remote_compact_task_inner( } attempt.track(sess.as_ref(), status, error.clone()).await; if let Err(err) = result { + sess.track_turn_codex_error(turn_context, &err); let event = EventMsg::Error( err.to_error_event(Some("Error running remote compact task".to_string())), ); diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rs index 0da3017f7..eb10eb211 100644 --- a/codex-rs/core/src/compact_remote_v2.rs +++ b/codex-rs/core/src/compact_remote_v2.rs @@ -155,6 +155,7 @@ async fn run_remote_compact_task_inner( } attempt.track(sess.as_ref(), status, error.clone()).await; if let Err(err) = result { + sess.track_turn_codex_error(turn_context, &err); let event = EventMsg::Error( err.to_error_event(Some("Error running remote compact task".to_string())), ); diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index d8f660e2a..7c49cd742 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -45,6 +45,7 @@ use chrono::Local; use chrono::Utc; use codex_analytics::AnalyticsEventsClient; use codex_analytics::SubAgentThreadStartedInput; +use codex_analytics::TurnCodexErrorFact; use codex_app_server_protocol::McpServerElicitationRequest; use codex_app_server_protocol::McpServerElicitationRequestParams; use codex_config::types::OAuthCredentialsStoreMode; @@ -1634,6 +1635,17 @@ impl Session { ) } + /// Record a terminal CodexErr before the app-server completion notification is reduced. + pub(crate) fn track_turn_codex_error(&self, turn_context: &TurnContext, error: &CodexErr) { + self.services + .analytics_events_client + .track_turn_codex_error(TurnCodexErrorFact::from_codex_err( + self.conversation_id.to_string(), + turn_context.sub_id.clone(), + error, + )); + } + /// Persist the event to rollout and send it to clients. pub(crate) async fn send_event(&self, turn_context: &TurnContext, msg: EventMsg) { let legacy_source = msg.clone(); diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index d1e9e59f7..62f8d9a49 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -369,7 +369,7 @@ pub(crate) async fn run_turn( // Aborted turn is reported via a different event. break; } - Err(CodexErr::InvalidImageRequest()) => { + Err(codex_error @ CodexErr::InvalidImageRequest()) => { { let mut state = sess.state.lock().await; error_or_panic( @@ -380,6 +380,7 @@ pub(crate) async fn run_turn( } } + sess.track_turn_codex_error(turn_context.as_ref(), &codex_error); let error = CodexErrorInfo::BadRequest; sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone()) .await; @@ -405,6 +406,7 @@ pub(crate) async fn run_turn( { warn!("failed to usage-limit active goal after usage-limit error: {err}"); } + sess.track_turn_codex_error(turn_context.as_ref(), &e); let event = EventMsg::Error(e.to_error_event(/*message_prefix*/ None)); sess.send_event(&turn_context, event).await; // let the user continue the conversation