mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
[codex-analytics] add turn tool counts to turn events (#21431)
## Summary - accumulate completed tool-item counts per turn from the item lifecycle - populate the reserved count fields on `codex_turn_event` - add reducer coverage for zero-count turns and mixed completed tool items ## Why PR #17090 moved tool-item analytics onto the item lifecycle, so the turn reducer can now derive the per-turn tool counts from the same completed items instead of leaving the reserved fields null. ## Validation - `just fmt` - `cargo test -p codex-analytics`
This commit is contained in:
rhan-oai
committed by
GitHub
Unverified
GitHub
parent
0dbad2a348
commit
cf6342b75b
@@ -69,15 +69,20 @@ use codex_app_server_protocol::ClientInfo;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ClientResponsePayload;
|
||||
use codex_app_server_protocol::CodexErrorInfo;
|
||||
use codex_app_server_protocol::CollabAgentTool;
|
||||
use codex_app_server_protocol::CollabAgentToolCallStatus;
|
||||
use codex_app_server_protocol::CommandAction;
|
||||
use codex_app_server_protocol::CommandExecutionSource;
|
||||
use codex_app_server_protocol::CommandExecutionStatus;
|
||||
use codex_app_server_protocol::DynamicToolCallStatus;
|
||||
use codex_app_server_protocol::InitializeCapabilities;
|
||||
use codex_app_server_protocol::InitializeParams;
|
||||
use codex_app_server_protocol::ItemCompletedNotification;
|
||||
use codex_app_server_protocol::ItemStartedNotification;
|
||||
use codex_app_server_protocol::JSONRPCErrorError;
|
||||
use codex_app_server_protocol::McpToolCallStatus;
|
||||
use codex_app_server_protocol::NonSteerableTurnKind;
|
||||
use codex_app_server_protocol::PatchApplyStatus;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use codex_app_server_protocol::SandboxPolicy as AppServerSandboxPolicy;
|
||||
use codex_app_server_protocol::ServerNotification;
|
||||
@@ -1709,6 +1714,14 @@ async fn item_lifecycle_notifications_publish_command_execution_event() {
|
||||
let mut events = Vec::new();
|
||||
|
||||
ingest_tool_review_prerequisites(&mut reducer, &mut events).await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(sample_turn_started_notification(
|
||||
"thread-1", "turn-1",
|
||||
))),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(ServerNotification::ItemStarted(
|
||||
@@ -2119,6 +2132,15 @@ async fn subagent_tool_items_inherit_parent_connection_metadata() {
|
||||
)
|
||||
.await;
|
||||
events.clear();
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(sample_turn_started_notification(
|
||||
"thread-subagent",
|
||||
"turn-subagent",
|
||||
))),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
@@ -2993,6 +3015,17 @@ async fn turn_lifecycle_emits_turn_event() {
|
||||
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
|
||||
assert_eq!(payload["event_params"]["status"], json!("completed"));
|
||||
assert_eq!(payload["event_params"]["steer_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["total_tool_call_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["shell_command_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["file_change_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["mcp_tool_call_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["dynamic_tool_call_count"], json!(0));
|
||||
assert_eq!(
|
||||
payload["event_params"]["subagent_tool_call_count"],
|
||||
json!(0)
|
||||
);
|
||||
assert_eq!(payload["event_params"]["web_search_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["image_generation_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["started_at"], json!(455));
|
||||
assert_eq!(payload["event_params"]["completed_at"], json!(456));
|
||||
assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
|
||||
@@ -3006,6 +3039,158 @@ async fn turn_lifecycle_emits_turn_event() {
|
||||
assert_eq!(payload["event_params"]["total_tokens"], json!(321));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_event_counts_completed_tool_items() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut out = Vec::new();
|
||||
|
||||
ingest_turn_prerequisites(
|
||||
&mut reducer,
|
||||
&mut out,
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ true,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
|
||||
let completed_tool_items = vec![
|
||||
sample_command_execution_item(CommandExecutionStatus::Completed, Some(0), Some(1)),
|
||||
ThreadItem::FileChange {
|
||||
id: "file-change-1".to_string(),
|
||||
changes: Vec::new(),
|
||||
status: PatchApplyStatus::Completed,
|
||||
},
|
||||
ThreadItem::McpToolCall {
|
||||
id: "mcp-1".to_string(),
|
||||
server: "server".to_string(),
|
||||
tool: "search".to_string(),
|
||||
status: McpToolCallStatus::Completed,
|
||||
arguments: json!({}),
|
||||
mcp_app_resource_uri: None,
|
||||
result: None,
|
||||
error: None,
|
||||
duration_ms: Some(2),
|
||||
},
|
||||
ThreadItem::DynamicToolCall {
|
||||
id: "dynamic-1".to_string(),
|
||||
namespace: None,
|
||||
tool: "render".to_string(),
|
||||
arguments: json!({}),
|
||||
status: DynamicToolCallStatus::Completed,
|
||||
content_items: None,
|
||||
success: Some(true),
|
||||
duration_ms: Some(3),
|
||||
},
|
||||
ThreadItem::CollabAgentToolCall {
|
||||
id: "collab-1".to_string(),
|
||||
tool: CollabAgentTool::SpawnAgent,
|
||||
status: CollabAgentToolCallStatus::Completed,
|
||||
sender_thread_id: "thread-2".to_string(),
|
||||
receiver_thread_ids: vec!["thread-child".to_string()],
|
||||
prompt: Some("help".to_string()),
|
||||
model: Some("gpt-5".to_string()),
|
||||
reasoning_effort: None,
|
||||
agents_states: Default::default(),
|
||||
},
|
||||
ThreadItem::WebSearch {
|
||||
id: "web-1".to_string(),
|
||||
query: "codex".to_string(),
|
||||
action: None,
|
||||
},
|
||||
ThreadItem::ImageGeneration {
|
||||
id: "image-1".to_string(),
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: None,
|
||||
result: "ok".to_string(),
|
||||
saved_path: None,
|
||||
},
|
||||
];
|
||||
|
||||
for item in completed_tool_items {
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(ServerNotification::ItemCompleted(
|
||||
ItemCompletedNotification {
|
||||
thread_id: "thread-2".to_string(),
|
||||
turn_id: "turn-2".to_string(),
|
||||
completed_at_ms: 1_000,
|
||||
item,
|
||||
},
|
||||
))),
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
|
||||
"thread-2",
|
||||
"turn-2",
|
||||
AppServerTurnStatus::Completed,
|
||||
/*codex_error_info*/ None,
|
||||
))),
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
let turn_event = out
|
||||
.iter()
|
||||
.find(|event| matches!(event, TrackEventRequest::TurnEvent(_)))
|
||||
.expect("turn event should be emitted");
|
||||
let payload = serde_json::to_value(turn_event).expect("serialize turn event");
|
||||
assert_eq!(payload["event_params"]["total_tool_call_count"], json!(7));
|
||||
assert_eq!(payload["event_params"]["shell_command_count"], json!(1));
|
||||
assert_eq!(payload["event_params"]["file_change_count"], json!(1));
|
||||
assert_eq!(payload["event_params"]["mcp_tool_call_count"], json!(1));
|
||||
assert_eq!(payload["event_params"]["dynamic_tool_call_count"], json!(1));
|
||||
assert_eq!(
|
||||
payload["event_params"]["subagent_tool_call_count"],
|
||||
json!(1)
|
||||
);
|
||||
assert_eq!(payload["event_params"]["web_search_count"], json!(1));
|
||||
assert_eq!(payload["event_params"]["image_generation_count"], json!(1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn item_completed_without_turn_state_does_not_create_turn_state() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut out = Vec::new();
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(ServerNotification::ItemCompleted(
|
||||
ItemCompletedNotification {
|
||||
thread_id: "thread-2".to_string(),
|
||||
turn_id: "turn-2".to_string(),
|
||||
completed_at_ms: 1_000,
|
||||
item: sample_command_execution_item(
|
||||
CommandExecutionStatus::Completed,
|
||||
Some(0),
|
||||
Some(1),
|
||||
),
|
||||
},
|
||||
))),
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
|
||||
"thread-2",
|
||||
"turn-2",
|
||||
AppServerTurnStatus::Completed,
|
||||
/*codex_error_info*/ None,
|
||||
))),
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepted_steers_increment_turn_steer_count() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
|
||||
@@ -794,8 +794,6 @@ pub(crate) struct CodexTurnEventParams {
|
||||
pub(crate) status: Option<TurnStatus>,
|
||||
pub(crate) turn_error: Option<CodexErrorInfo>,
|
||||
pub(crate) steer_count: Option<usize>,
|
||||
// TODO(rhan-oai): Populate these once tool-call accounting is emitted from
|
||||
// core; the schema is reserved but these fields are currently always None.
|
||||
pub(crate) total_tool_call_count: Option<usize>,
|
||||
pub(crate) shell_command_count: Option<usize>,
|
||||
pub(crate) file_change_count: Option<usize>,
|
||||
|
||||
@@ -271,6 +271,7 @@ struct TurnState {
|
||||
completed: Option<CompletedTurnState>,
|
||||
latest_diff: Option<String>,
|
||||
steer_count: usize,
|
||||
tool_counts: TurnToolCounts,
|
||||
}
|
||||
|
||||
#[derive(Hash, Eq, PartialEq)]
|
||||
@@ -280,6 +281,42 @@ struct ToolItemKey {
|
||||
item_id: String,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TurnToolCounts {
|
||||
total: usize,
|
||||
shell_command: usize,
|
||||
file_change: usize,
|
||||
mcp_tool_call: usize,
|
||||
dynamic_tool_call: usize,
|
||||
subagent_tool_call: usize,
|
||||
web_search: usize,
|
||||
image_generation: usize,
|
||||
}
|
||||
|
||||
impl TurnToolCounts {
|
||||
fn record(&mut self, item: &ThreadItem) {
|
||||
match item {
|
||||
ThreadItem::CommandExecution { .. } => self.shell_command += 1,
|
||||
ThreadItem::FileChange { .. } => self.file_change += 1,
|
||||
ThreadItem::McpToolCall { .. } => self.mcp_tool_call += 1,
|
||||
ThreadItem::DynamicToolCall { .. } => self.dynamic_tool_call += 1,
|
||||
ThreadItem::CollabAgentToolCall { .. } => self.subagent_tool_call += 1,
|
||||
ThreadItem::WebSearch { .. } => self.web_search += 1,
|
||||
ThreadItem::ImageGeneration { .. } => self.image_generation += 1,
|
||||
ThreadItem::UserMessage { .. }
|
||||
| ThreadItem::HookPrompt { .. }
|
||||
| ThreadItem::AgentMessage { .. }
|
||||
| ThreadItem::Plan { .. }
|
||||
| ThreadItem::Reasoning { .. }
|
||||
| ThreadItem::ImageView { .. }
|
||||
| ThreadItem::EnteredReviewMode { .. }
|
||||
| ThreadItem::ExitedReviewMode { .. }
|
||||
| ThreadItem::ContextCompaction { .. } => return,
|
||||
}
|
||||
self.total += 1;
|
||||
}
|
||||
}
|
||||
|
||||
impl AnalyticsReducer {
|
||||
pub(crate) async fn ingest(&mut self, input: AnalyticsFact, out: &mut Vec<TrackEventRequest>) {
|
||||
match input {
|
||||
@@ -497,6 +534,7 @@ impl AnalyticsReducer {
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
turn_state.thread_id = Some(thread_id);
|
||||
turn_state.num_input_images = Some(num_input_images);
|
||||
@@ -520,6 +558,7 @@ impl AnalyticsReducer {
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
turn_state.thread_id = Some(input.thread_id);
|
||||
turn_state.token_usage = Some(input.token_usage);
|
||||
@@ -684,6 +723,7 @@ impl AnalyticsReducer {
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
turn_state.connection_id = Some(connection_id);
|
||||
turn_state.thread_id = Some(pending_request.thread_id);
|
||||
@@ -777,6 +817,16 @@ impl AnalyticsReducer {
|
||||
let Some(item_id) = tracked_tool_item_id(¬ification.item) else {
|
||||
return;
|
||||
};
|
||||
let Some(turn_state) = self.turns.get_mut(¬ification.turn_id) else {
|
||||
tracing::warn!(
|
||||
thread_id = %notification.thread_id,
|
||||
turn_id = %notification.turn_id,
|
||||
item_id,
|
||||
"dropping turn tool count update: missing turn state"
|
||||
);
|
||||
return;
|
||||
};
|
||||
turn_state.tool_counts.record(¬ification.item);
|
||||
let key = ToolItemKey {
|
||||
thread_id: notification.thread_id.clone(),
|
||||
turn_id: notification.turn_id.clone(),
|
||||
@@ -823,6 +873,7 @@ impl AnalyticsReducer {
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
turn_state.started_at = notification
|
||||
.turn
|
||||
@@ -843,6 +894,7 @@ impl AnalyticsReducer {
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
turn_state.thread_id = Some(notification.thread_id);
|
||||
turn_state.latest_diff = Some(notification.diff);
|
||||
@@ -861,6 +913,7 @@ impl AnalyticsReducer {
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
turn_state.completed = Some(CompletedTurnState {
|
||||
status: analytics_turn_status(notification.turn.status),
|
||||
@@ -1776,14 +1829,14 @@ fn codex_turn_event_params(
|
||||
status: completed.status,
|
||||
turn_error: completed.turn_error,
|
||||
steer_count: Some(turn_state.steer_count),
|
||||
total_tool_call_count: None,
|
||||
shell_command_count: None,
|
||||
file_change_count: None,
|
||||
mcp_tool_call_count: None,
|
||||
dynamic_tool_call_count: None,
|
||||
subagent_tool_call_count: None,
|
||||
web_search_count: None,
|
||||
image_generation_count: None,
|
||||
total_tool_call_count: Some(turn_state.tool_counts.total),
|
||||
shell_command_count: Some(turn_state.tool_counts.shell_command),
|
||||
file_change_count: Some(turn_state.tool_counts.file_change),
|
||||
mcp_tool_call_count: Some(turn_state.tool_counts.mcp_tool_call),
|
||||
dynamic_tool_call_count: Some(turn_state.tool_counts.dynamic_tool_call),
|
||||
subagent_tool_call_count: Some(turn_state.tool_counts.subagent_tool_call),
|
||||
web_search_count: Some(turn_state.tool_counts.web_search),
|
||||
image_generation_count: Some(turn_state.tool_counts.image_generation),
|
||||
input_tokens: token_usage
|
||||
.as_ref()
|
||||
.map(|token_usage| token_usage.input_tokens),
|
||||
|
||||
Reference in New Issue
Block a user