mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
[codex] Add turn profiling analytics (#26484)
## Summary Add flat profiling fields to `codex_turn_event` so analytics can explain where turn wall-clock time is spent without changing tool execution behavior. The profile reports: - time before the first sampling request - sampling time across all attempts and follow-ups - overhead between sampling requests - time blocked in the post-sampling tool drain - time after the final sampling request - sampling request and retry counts ## Implementation - Extend the existing turn timing state with constant-memory phase accounting and one RAII phase guard. - Observe sampling and the existing post-sampling drain only at turn orchestration boundaries. - Keep tool runtime, tool futures, response item handling, and turn lifecycle values unchanged. - Add the profiling fields directly to the existing analytics turn event without changing app-server protocol or rollout persistence. - Use the existing turn `status` to distinguish completed, failed, and interrupted profiles. Exact sampling/tool overlap is intentionally omitted because measuring tool completion accurately would require hooks in the tool execution path. ## Validation - Add app-server end-to-end coverage for a single-sampling turn with no blocking tool work. - Add app-server end-to-end coverage for `request_user_input` blocking followed by a second sampling request. - CI is running on the PR; tests were not executed locally per repository guidance.
This commit is contained in:
committed by
GitHub
Unverified
parent
82b15b65e2
commit
8d72fb6de9
@@ -72,6 +72,8 @@ use crate::facts::SubAgentThreadStartedInput;
|
||||
use crate::facts::ThreadInitializationMode;
|
||||
use crate::facts::TurnCodexError;
|
||||
use crate::facts::TurnCodexErrorFact;
|
||||
use crate::facts::TurnProfile;
|
||||
use crate::facts::TurnProfileFact;
|
||||
use crate::facts::TurnResolvedConfigFact;
|
||||
use crate::facts::TurnStatus;
|
||||
use crate::facts::TurnSteerRejectionReason;
|
||||
@@ -316,6 +318,7 @@ struct CompletedTurnState {
|
||||
duration_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TurnState {
|
||||
connection_id: Option<u64>,
|
||||
thread_id: Option<String>,
|
||||
@@ -323,6 +326,7 @@ struct TurnState {
|
||||
resolved_config: Option<TurnResolvedConfigFact>,
|
||||
started_at: Option<u64>,
|
||||
token_usage: Option<TokenUsage>,
|
||||
profile: Option<TurnProfile>,
|
||||
completed: Option<CompletedTurnState>,
|
||||
codex_error: Option<TurnCodexError>,
|
||||
latest_diff: Option<String>,
|
||||
@@ -464,6 +468,9 @@ impl AnalyticsReducer {
|
||||
CustomAnalyticsFact::TurnTokenUsage(input) => {
|
||||
self.ingest_turn_token_usage(*input, out).await;
|
||||
}
|
||||
CustomAnalyticsFact::TurnProfile(input) => {
|
||||
self.ingest_turn_profile(*input, out).await;
|
||||
}
|
||||
CustomAnalyticsFact::TurnCodexError(input) => {
|
||||
self.ingest_turn_codex_error(*input);
|
||||
}
|
||||
@@ -604,19 +611,7 @@ impl AnalyticsReducer {
|
||||
let turn_id = input.turn_id.clone();
|
||||
let thread_id = input.thread_id.clone();
|
||||
let num_input_images = input.num_input_images;
|
||||
let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState {
|
||||
connection_id: None,
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
completed: None,
|
||||
codex_error: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
let turn_state = self.turns.entry(turn_id.clone()).or_default();
|
||||
turn_state.thread_id = Some(thread_id);
|
||||
turn_state.num_input_images = Some(num_input_images);
|
||||
turn_state.resolved_config = Some(input);
|
||||
@@ -629,43 +624,30 @@ impl AnalyticsReducer {
|
||||
out: &mut Vec<TrackEventRequest>,
|
||||
) {
|
||||
let turn_id = input.turn_id.clone();
|
||||
let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState {
|
||||
connection_id: None,
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
completed: None,
|
||||
codex_error: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
let turn_state = self.turns.entry(turn_id.clone()).or_default();
|
||||
turn_state.thread_id = Some(input.thread_id);
|
||||
turn_state.token_usage = Some(input.token_usage);
|
||||
self.maybe_emit_turn_event(&turn_id, out).await;
|
||||
}
|
||||
|
||||
async fn ingest_turn_profile(
|
||||
&mut self,
|
||||
input: TurnProfileFact,
|
||||
out: &mut Vec<TrackEventRequest>,
|
||||
) {
|
||||
let TurnProfileFact { turn_id, profile } = input;
|
||||
let turn_state = self.turns.entry(turn_id.clone()).or_default();
|
||||
turn_state.profile = Some(profile);
|
||||
self.maybe_emit_turn_event(&turn_id, out).await;
|
||||
}
|
||||
|
||||
fn ingest_turn_codex_error(&mut self, input: TurnCodexErrorFact) {
|
||||
let TurnCodexErrorFact {
|
||||
turn_id,
|
||||
thread_id,
|
||||
error,
|
||||
} = input;
|
||||
let turn_state = self.turns.entry(turn_id).or_insert(TurnState {
|
||||
connection_id: None,
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
completed: None,
|
||||
codex_error: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
let turn_state = self.turns.entry(turn_id).or_default();
|
||||
turn_state.thread_id.get_or_insert(thread_id);
|
||||
turn_state.codex_error = Some(error);
|
||||
}
|
||||
@@ -818,19 +800,7 @@ impl AnalyticsReducer {
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState {
|
||||
connection_id: None,
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
completed: None,
|
||||
codex_error: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
let turn_state = self.turns.entry(turn_id.clone()).or_default();
|
||||
turn_state.connection_id = Some(connection_id);
|
||||
turn_state.thread_id = Some(pending_request.thread_id);
|
||||
turn_state.num_input_images = Some(pending_request.num_input_images);
|
||||
@@ -1178,61 +1148,19 @@ impl AnalyticsReducer {
|
||||
self.ingest_guardian_review_completed(notification, out);
|
||||
}
|
||||
ServerNotification::TurnStarted(notification) => {
|
||||
let turn_state = self.turns.entry(notification.turn.id).or_insert(TurnState {
|
||||
connection_id: None,
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
completed: None,
|
||||
codex_error: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
let turn_state = self.turns.entry(notification.turn.id).or_default();
|
||||
turn_state.started_at = notification
|
||||
.turn
|
||||
.started_at
|
||||
.and_then(|started_at| u64::try_from(started_at).ok());
|
||||
}
|
||||
ServerNotification::TurnDiffUpdated(notification) => {
|
||||
let turn_state =
|
||||
self.turns
|
||||
.entry(notification.turn_id.clone())
|
||||
.or_insert(TurnState {
|
||||
connection_id: None,
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
completed: None,
|
||||
codex_error: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
let turn_state = self.turns.entry(notification.turn_id.clone()).or_default();
|
||||
turn_state.thread_id = Some(notification.thread_id);
|
||||
turn_state.latest_diff = Some(notification.diff);
|
||||
}
|
||||
ServerNotification::TurnCompleted(notification) => {
|
||||
let turn_state =
|
||||
self.turns
|
||||
.entry(notification.turn.id.clone())
|
||||
.or_insert(TurnState {
|
||||
connection_id: None,
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
completed: None,
|
||||
codex_error: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
let turn_state = self.turns.entry(notification.turn.id.clone()).or_default();
|
||||
turn_state.completed = Some(CompletedTurnState {
|
||||
status: analytics_turn_status(notification.turn.status),
|
||||
turn_error: notification
|
||||
@@ -1511,6 +1439,7 @@ impl AnalyticsReducer {
|
||||
if turn_state.thread_id.is_none()
|
||||
|| turn_state.num_input_images.is_none()
|
||||
|| turn_state.resolved_config.is_none()
|
||||
|| turn_state.profile.is_none()
|
||||
|| turn_state.completed.is_none()
|
||||
{
|
||||
return;
|
||||
@@ -2457,12 +2386,20 @@ fn codex_turn_event_params(
|
||||
turn_state: &TurnState,
|
||||
thread_metadata: &ThreadMetadataState,
|
||||
) -> CodexTurnEventParams {
|
||||
let (Some(thread_id), Some(num_input_images), Some(resolved_config), Some(completed)) = (
|
||||
let (
|
||||
Some(thread_id),
|
||||
Some(num_input_images),
|
||||
Some(resolved_config),
|
||||
Some(profile),
|
||||
Some(completed),
|
||||
) = (
|
||||
turn_state.thread_id.clone(),
|
||||
turn_state.num_input_images,
|
||||
turn_state.resolved_config.clone(),
|
||||
turn_state.profile.clone(),
|
||||
turn_state.completed.clone(),
|
||||
) else {
|
||||
)
|
||||
else {
|
||||
unreachable!("turn event params require a fully populated turn state");
|
||||
};
|
||||
let started_at = turn_state.started_at;
|
||||
@@ -2488,6 +2425,15 @@ fn codex_turn_event_params(
|
||||
workspace_kind,
|
||||
is_first_turn,
|
||||
} = resolved_config;
|
||||
let TurnProfile {
|
||||
before_first_sampling_ms,
|
||||
sampling_ms,
|
||||
between_sampling_overhead_ms,
|
||||
tool_blocking_ms,
|
||||
after_last_sampling_ms,
|
||||
sampling_request_count,
|
||||
sampling_retry_count,
|
||||
} = profile;
|
||||
let token_usage = turn_state.token_usage.clone();
|
||||
let codex_error = turn_state.codex_error.as_ref();
|
||||
CodexTurnEventParams {
|
||||
@@ -2550,6 +2496,13 @@ fn codex_turn_event_params(
|
||||
total_tokens: token_usage
|
||||
.as_ref()
|
||||
.map(|token_usage| token_usage.total_tokens),
|
||||
before_first_sampling_ms,
|
||||
sampling_ms,
|
||||
between_sampling_overhead_ms,
|
||||
tool_blocking_ms,
|
||||
after_last_sampling_ms,
|
||||
sampling_request_count,
|
||||
sampling_retry_count,
|
||||
duration_ms: completed.duration_ms,
|
||||
started_at,
|
||||
completed_at: Some(completed.completed_at),
|
||||
|
||||
Reference in New Issue
Block a user