agent/codex
2
0
Fork 0
mirror of https://github.com/pchuan98/codex.git synced 2026-07-01 00:31:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

[codex-analytics] Track CodexErr details in turn analytics (#25707)

Browse Source 
## Summary
- add analytics-only `CodexErr` telemetry to `codex_turn_event` while
leaving existing `turn_error` unchanged
- record terminal `CodexErr` facts from core immediately before the
existing turn error event is sent
- emit source-truth `codex_error_*` fields for downstream analytics,
including the raw `CodexErr::InvalidRequest(String)` message as
`codex_error_subreason`

## Validation
- `just test -p codex-analytics`
- attempted `just test -p codex-core`, but the local run timed out
across unrelated integration suites in this environment and is not being
used as validation
This commit is contained in:
rhan-oai
2026-06-02 11:40:35 -07:00
committed by GitHub
Unverified
parent 503ec190a8
commit 8e4b92d294
11 changed files with 249 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/analytics/src/analytics_client_tests.rs
+33
View File
@@ -62,6 +62,7 @@ use crate::facts::SkillInvokedInput;
use crate::facts::SubAgentThreadStartedInput;
use crate::facts::ThreadInitializationMode;
use crate::facts::TrackEventsContext;
use crate::facts::TurnCodexErrorFact;
use crate::facts::TurnResolvedConfigFact;
use crate::facts::TurnStatus;
use crate::facts::TurnSteerRequestError;
@@ -132,6 +133,7 @@ use codex_plugin::PluginTelemetryMetadata;
use codex_protocol::approvals::NetworkApprovalProtocol;
use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::config_types::ModeKind;
use codex_protocol::error::CodexErr;
use codex_protocol::models::NetworkPermissions as CoreNetworkPermissions;
use codex_protocol::models::PermissionProfile as CorePermissionProfile;
use codex_protocol::protocol::AskForApproval;
@@ -3256,6 +3258,9 @@ fn turn_event_serializes_expected_shape() {
is_first_turn: true,
status: Some(TurnStatus::Completed),
turn_error: None,
codex_error_kind: None,
codex_error_subreason: None,
codex_error_http_status_code: None,
steer_count: Some(0),
total_tool_call_count: None,
shell_command_count: None,
@@ -3318,6 +3323,9 @@ fn turn_event_serializes_expected_shape() {
"is_first_turn": true,
"status": "completed",
"turn_error": null,
"codex_error_kind": null,
"codex_error_subreason": null,
"codex_error_http_status_code": null,
"steer_count": 0,
"total_tool_call_count": null,
"shell_command_count": null,
@@ -3983,6 +3991,18 @@ async fn turn_lifecycle_emits_failed_turn_event() {
/*include_token_usage*/ false,
)
.await;
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::TurnCodexError(Box::new(
TurnCodexErrorFact::from_codex_err(
"thread-2".to_string(),
"turn-2".to_string(),
&CodexErr::InvalidRequest("unknown turn environment id `env-2`".to_string()),
),
))),
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
@@ -3999,6 +4019,18 @@ async fn turn_lifecycle_emits_failed_turn_event() {
let payload = serde_json::to_value(&out[0]).expect("serialize turn event");
assert_eq!(payload["event_params"]["status"], json!("failed"));
assert_eq!(payload["event_params"]["turn_error"], json!("badRequest"));
assert_eq!(
payload["event_params"]["codex_error_kind"],
json!("invalid_request")
);
assert_eq!(
payload["event_params"]["codex_error_subreason"],
json!("unknown turn environment id `env-2`")
);
assert_eq!(
payload["event_params"]["codex_error_http_status_code"],
json!(null)
);
}
#[tokio::test]
@@ -4031,6 +4063,7 @@ async fn turn_lifecycle_emits_interrupted_turn_event_without_error() {
let payload = serde_json::to_value(&out[0]).expect("serialize turn event");
assert_eq!(payload["event_params"]["status"], json!("interrupted"));
assert_eq!(payload["event_params"]["turn_error"], json!(null));
assert_eq!(payload["event_params"]["codex_error_kind"], json!(null));
}
#[tokio::test]
codex-rs/analytics/src/client.rs
+7
View File
@@ -18,6 +18,7 @@ use crate::facts::SkillInvocation;
use crate::facts::SkillInvokedInput;
use crate::facts::SubAgentThreadStartedInput;
use crate::facts::TrackEventsContext;
use crate::facts::TurnCodexErrorFact;
use crate::facts::TurnResolvedConfigFact;
use crate::facts::TurnTokenUsageFact;
use crate::reducer::AnalyticsReducer;
@@ -256,6 +257,12 @@ impl AnalyticsEventsClient {
)));
}
pub fn track_turn_codex_error(&self, fact: TurnCodexErrorFact) {
self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::TurnCodexError(
Box::new(fact),
)));
}
pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput {
codex-rs/analytics/src/events.rs
+4
View File
@@ -3,6 +3,7 @@ use std::time::Instant;
use crate::facts::AcceptedLineFingerprint;
use crate::facts::AppInvocation;
use crate::facts::CodexCompactionEvent;
use crate::facts::CodexErrKind;
use crate::facts::CompactionImplementation;
use crate::facts::CompactionPhase;
use crate::facts::CompactionReason;
@@ -797,6 +798,9 @@ pub(crate) struct CodexTurnEventParams {
pub(crate) is_first_turn: bool,
pub(crate) status: Option<TurnStatus>,
pub(crate) turn_error: Option<CodexErrorInfo>,
pub(crate) codex_error_kind: Option<CodexErrKind>,
pub(crate) codex_error_subreason: Option<String>,
pub(crate) codex_error_http_status_code: Option<u16>,
pub(crate) steer_count: Option<usize>,
pub(crate) total_tool_call_count: Option<usize>,
pub(crate) shell_command_count: Option<usize>,
codex-rs/analytics/src/facts.rs
+146
View File
@@ -15,6 +15,7 @@ use codex_protocol::config_types::ModeKind;
use codex_protocol::config_types::Personality;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::config_types::ServiceTier;
use codex_protocol::error::CodexErr;
use codex_protocol::models::PermissionProfile;
use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::protocol::AskForApproval;
@@ -29,6 +30,9 @@ use codex_protocol::request_permissions::RequestPermissionsResponse;
use serde::Serialize;
use std::path::PathBuf;
const INVALID_REQUEST_SUBREASON_MAX_BYTES: usize = 512;
const INVALID_REQUEST_SUBREASON_TRUNCATION_SUFFIX: &str = "...";
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct AcceptedLineFingerprint {
pub path_hash: String,
@@ -99,6 +103,147 @@ pub struct TurnTokenUsageFact {
pub token_usage: TokenUsage,
}
#[derive(Clone)]
pub struct TurnCodexErrorFact {
pub(crate) turn_id: String,
pub(crate) thread_id: String,
pub(crate) error: TurnCodexError,
}
impl TurnCodexErrorFact {
pub fn from_codex_err(thread_id: String, turn_id: String, error: &CodexErr) -> Self {
Self {
turn_id,
thread_id,
error: TurnCodexError::from_codex_err(error),
}
}
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum CodexErrKind {
TurnAborted,
Stream,
ContextWindowExceeded,
ThreadNotFound,
AgentLimitReached,
SessionConfiguredNotFirstEvent,
Timeout,
RequestTimeout,
Spawn,
Interrupted,
UnexpectedStatus,
InvalidRequest,
InvalidImageRequest,
UsageLimitReached,
ServerOverloaded,
CyberPolicy,
ResponseStreamFailed,
ConnectionFailed,
QuotaExceeded,
UsageNotIncluded,
InternalServerError,
RetryLimit,
InternalAgentDied,
Sandbox,
LandlockSandboxExecutableNotProvided,
UnsupportedOperation,
RefreshTokenFailed,
Fatal,
Io,
Json,
#[cfg(target_os = "linux")]
LandlockRuleset,
#[cfg(target_os = "linux")]
LandlockPathFd,
TokioJoin,
EnvVar,
}
#[derive(Clone)]
pub(crate) struct TurnCodexError {
pub(crate) kind: CodexErrKind,
pub(crate) subreason: Option<String>,
pub(crate) http_status_code: Option<u16>,
}
impl TurnCodexError {
fn from_codex_err(error: &CodexErr) -> Self {
Self {
kind: error.into(),
subreason: match error {
CodexErr::InvalidRequest(message) => {
// InvalidRequest can contain raw provider response bodies, so bound the
// analytics copy without changing the source CodexErr.
let subreason = if message.len() <= INVALID_REQUEST_SUBREASON_MAX_BYTES {
message.clone()
} else {
let truncated_len = message.floor_char_boundary(
INVALID_REQUEST_SUBREASON_MAX_BYTES
.saturating_sub(INVALID_REQUEST_SUBREASON_TRUNCATION_SUFFIX.len()),
);
format!(
"{}{INVALID_REQUEST_SUBREASON_TRUNCATION_SUFFIX}",
&message[..truncated_len]
)
};
Some(subreason)
}
_ => None,
},
http_status_code: error.http_status_code_value(),
}
}
}
impl From<&CodexErr> for CodexErrKind {
fn from(error: &CodexErr) -> Self {
match error {
CodexErr::TurnAborted => CodexErrKind::TurnAborted,
CodexErr::Stream(..) => CodexErrKind::Stream,
CodexErr::ContextWindowExceeded => CodexErrKind::ContextWindowExceeded,
CodexErr::ThreadNotFound(_) => CodexErrKind::ThreadNotFound,
CodexErr::AgentLimitReached { .. } => CodexErrKind::AgentLimitReached,
CodexErr::SessionConfiguredNotFirstEvent => {
CodexErrKind::SessionConfiguredNotFirstEvent
}
CodexErr::Timeout => CodexErrKind::Timeout,
CodexErr::RequestTimeout => CodexErrKind::RequestTimeout,
CodexErr::Spawn => CodexErrKind::Spawn,
CodexErr::Interrupted => CodexErrKind::Interrupted,
CodexErr::UnexpectedStatus(_) => CodexErrKind::UnexpectedStatus,
CodexErr::InvalidRequest(_) => CodexErrKind::InvalidRequest,
CodexErr::InvalidImageRequest() => CodexErrKind::InvalidImageRequest,
CodexErr::UsageLimitReached(_) => CodexErrKind::UsageLimitReached,
CodexErr::ServerOverloaded => CodexErrKind::ServerOverloaded,
CodexErr::CyberPolicy { .. } => CodexErrKind::CyberPolicy,
CodexErr::ResponseStreamFailed(_) => CodexErrKind::ResponseStreamFailed,
CodexErr::ConnectionFailed(_) => CodexErrKind::ConnectionFailed,
CodexErr::QuotaExceeded => CodexErrKind::QuotaExceeded,
CodexErr::UsageNotIncluded => CodexErrKind::UsageNotIncluded,
CodexErr::InternalServerError => CodexErrKind::InternalServerError,
CodexErr::RetryLimit(_) => CodexErrKind::RetryLimit,
CodexErr::InternalAgentDied => CodexErrKind::InternalAgentDied,
CodexErr::Sandbox(_) => CodexErrKind::Sandbox,
CodexErr::LandlockSandboxExecutableNotProvided => {
CodexErrKind::LandlockSandboxExecutableNotProvided
}
CodexErr::UnsupportedOperation(_) => CodexErrKind::UnsupportedOperation,
CodexErr::RefreshTokenFailed(_) => CodexErrKind::RefreshTokenFailed,
CodexErr::Fatal(_) => CodexErrKind::Fatal,
CodexErr::Io(_) => CodexErrKind::Io,
CodexErr::Json(_) => CodexErrKind::Json,
#[cfg(target_os = "linux")]
CodexErr::LandlockRuleset(_) => CodexErrKind::LandlockRuleset,
#[cfg(target_os = "linux")]
CodexErr::LandlockPathFd(_) => CodexErrKind::LandlockPathFd,
CodexErr::TokioJoin(_) => CodexErrKind::TokioJoin,
CodexErr::EnvVar(_) => CodexErrKind::EnvVar,
}
}
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TurnStatus {
@@ -329,6 +474,7 @@ pub(crate) enum CustomAnalyticsFact {
GuardianReview(Box<GuardianReviewEventParams>),
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
TurnTokenUsage(Box<TurnTokenUsageFact>),
TurnCodexError(Box<TurnCodexErrorFact>),
SkillInvoked(SkillInvokedInput),
AppMentioned(AppMentionedInput),
AppUsed(AppUsedInput),
codex-rs/analytics/src/lib.rs
+1
View File
@@ -38,6 +38,7 @@ pub use facts::SkillInvocation;
pub use facts::SubAgentThreadStartedInput;
pub use facts::ThreadInitializationMode;
pub use facts::TrackEventsContext;
pub use facts::TurnCodexErrorFact;
pub use facts::TurnResolvedConfigFact;
pub use facts::TurnStatus;
pub use facts::TurnSteerRejectionReason;
codex-rs/analytics/src/reducer.rs
+39
View File
@@ -70,6 +70,8 @@ use crate::facts::PluginUsedInput;
use crate::facts::SkillInvokedInput;
use crate::facts::SubAgentThreadStartedInput;
use crate::facts::ThreadInitializationMode;
use crate::facts::TurnCodexError;
use crate::facts::TurnCodexErrorFact;
use crate::facts::TurnResolvedConfigFact;
use crate::facts::TurnStatus;
use crate::facts::TurnSteerRejectionReason;
@@ -322,6 +324,7 @@ struct TurnState {
started_at: Option<u64>,
token_usage: Option<TokenUsage>,
completed: Option<CompletedTurnState>,
codex_error: Option<TurnCodexError>,
latest_diff: Option<String>,
steer_count: usize,
tool_counts: TurnToolCounts,
@@ -461,6 +464,9 @@ impl AnalyticsReducer {
CustomAnalyticsFact::TurnTokenUsage(input) => {
self.ingest_turn_token_usage(*input, out).await;
}
CustomAnalyticsFact::TurnCodexError(input) => {
self.ingest_turn_codex_error(*input);
}
CustomAnalyticsFact::SkillInvoked(input) => {
self.ingest_skill_invoked(input, out).await;
}
@@ -606,6 +612,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
codex_error: None,
latest_diff: None,
steer_count: 0,
tool_counts: TurnToolCounts::default(),
@@ -630,6 +637,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
codex_error: None,
latest_diff: None,
steer_count: 0,
tool_counts: TurnToolCounts::default(),
@@ -639,6 +647,29 @@ impl AnalyticsReducer {
self.maybe_emit_turn_event(&turn_id, out).await;
}
fn ingest_turn_codex_error(&mut self, input: TurnCodexErrorFact) {
let TurnCodexErrorFact {
turn_id,
thread_id,
error,
} = input;
let turn_state = self.turns.entry(turn_id).or_insert(TurnState {
connection_id: None,
thread_id: None,
num_input_images: None,
resolved_config: None,
started_at: None,
token_usage: None,
completed: None,
codex_error: None,
latest_diff: None,
steer_count: 0,
tool_counts: TurnToolCounts::default(),
});
turn_state.thread_id.get_or_insert(thread_id);
turn_state.codex_error = Some(error);
}
async fn ingest_skill_invoked(
&mut self,
input: SkillInvokedInput,
@@ -795,6 +826,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
codex_error: None,
latest_diff: None,
steer_count: 0,
tool_counts: TurnToolCounts::default(),
@@ -1154,6 +1186,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
codex_error: None,
latest_diff: None,
steer_count: 0,
tool_counts: TurnToolCounts::default(),
@@ -1175,6 +1208,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
codex_error: None,
latest_diff: None,
steer_count: 0,
tool_counts: TurnToolCounts::default(),
@@ -1194,6 +1228,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
codex_error: None,
latest_diff: None,
steer_count: 0,
tool_counts: TurnToolCounts::default(),
@@ -2451,6 +2486,7 @@ fn codex_turn_event_params(
is_first_turn,
} = resolved_config;
let token_usage = turn_state.token_usage.clone();
let codex_error = turn_state.codex_error.as_ref();
CodexTurnEventParams {
thread_id,
session_id: thread_metadata.session_id.clone(),
@@ -2483,6 +2519,9 @@ fn codex_turn_event_params(
is_first_turn,
status: completed.status,
turn_error: completed.turn_error,
codex_error_kind: codex_error.map(|error| error.kind),
codex_error_subreason: codex_error.and_then(|error| error.subreason.clone()),
codex_error_http_status_code: codex_error.and_then(|error| error.http_status_code),
steer_count: Some(turn_state.steer_count),
total_tool_call_count: Some(turn_state.tool_counts.total),
shell_command_count: Some(turn_state.tool_counts.shell_command),
codex-rs/core/src/compact.rs
+2
View File
@@ -240,6 +240,7 @@ async fn run_compact_task_inner_impl(
continue;
}
sess.set_total_tokens_full(turn_context.as_ref()).await;
sess.track_turn_codex_error(turn_context.as_ref(), &e);
let event = EventMsg::Error(e.to_error_event(/*message_prefix*/ None));
sess.send_event(&turn_context, event).await;
return Err(e);
@@ -257,6 +258,7 @@ async fn run_compact_task_inner_impl(
tokio::time::sleep(delay).await;
continue;
} else {
sess.track_turn_codex_error(turn_context.as_ref(), &e);
let event = EventMsg::Error(e.to_error_event(/*message_prefix*/ None));
sess.send_event(&turn_context, event).await;
return Err(e);
codex-rs/core/src/compact_remote.rs
+1
View File
@@ -138,6 +138,7 @@ async fn run_remote_compact_task_inner(
}
attempt.track(sess.as_ref(), status, error.clone()).await;
if let Err(err) = result {
sess.track_turn_codex_error(turn_context, &err);
let event = EventMsg::Error(
err.to_error_event(Some("Error running remote compact task".to_string())),
);
codex-rs/core/src/compact_remote_v2.rs
+1
View File
@@ -155,6 +155,7 @@ async fn run_remote_compact_task_inner(
}
attempt.track(sess.as_ref(), status, error.clone()).await;
if let Err(err) = result {
sess.track_turn_codex_error(turn_context, &err);
let event = EventMsg::Error(
err.to_error_event(Some("Error running remote compact task".to_string())),
);
codex-rs/core/src/session/mod.rs
+12
View File
@@ -45,6 +45,7 @@ use chrono::Local;
use chrono::Utc;
use codex_analytics::AnalyticsEventsClient;
use codex_analytics::SubAgentThreadStartedInput;
use codex_analytics::TurnCodexErrorFact;
use codex_app_server_protocol::McpServerElicitationRequest;
use codex_app_server_protocol::McpServerElicitationRequestParams;
use codex_config::types::OAuthCredentialsStoreMode;
@@ -1634,6 +1635,17 @@ impl Session {
)
}
/// Record a terminal CodexErr before the app-server completion notification is reduced.
pub(crate) fn track_turn_codex_error(&self, turn_context: &TurnContext, error: &CodexErr) {
self.services
.analytics_events_client
.track_turn_codex_error(TurnCodexErrorFact::from_codex_err(
self.conversation_id.to_string(),
turn_context.sub_id.clone(),
error,
));
}
/// Persist the event to rollout and send it to clients.
pub(crate) async fn send_event(&self, turn_context: &TurnContext, msg: EventMsg) {
let legacy_source = msg.clone();
codex-rs/core/src/session/turn.rs
+3 -1
View File
@@ -369,7 +369,7 @@ pub(crate) async fn run_turn(
// Aborted turn is reported via a different event.
break;
}
Err(CodexErr::InvalidImageRequest()) => {
Err(codex_error @ CodexErr::InvalidImageRequest()) => {
{
let mut state = sess.state.lock().await;
error_or_panic(
@@ -380,6 +380,7 @@ pub(crate) async fn run_turn(
}
}
sess.track_turn_codex_error(turn_context.as_ref(), &codex_error);
let error = CodexErrorInfo::BadRequest;
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
.await;
@@ -405,6 +406,7 @@ pub(crate) async fn run_turn(
{
warn!("failed to usage-limit active goal after usage-limit error: {err}");
}
sess.track_turn_codex_error(turn_context.as_ref(), &e);
let event = EventMsg::Error(e.to_error_event(/*message_prefix*/ None));
sess.send_event(&turn_context, event).await;
// let the user continue the conversation