[codex] Add turn profiling analytics (#26484)

## Summary

Add flat profiling fields to `codex_turn_event` so analytics can explain
where turn wall-clock time is spent without changing tool execution
behavior.

The profile reports:
- time before the first sampling request
- sampling time across all attempts and follow-ups
- overhead between sampling requests
- time blocked in the post-sampling tool drain
- time after the final sampling request
- sampling request and retry counts

## Implementation

- Extend the existing turn timing state with constant-memory phase
accounting and one RAII phase guard.
- Observe sampling and the existing post-sampling drain only at turn
orchestration boundaries.
- Keep tool runtime, tool futures, response item handling, and turn
lifecycle values unchanged.
- Add the profiling fields directly to the existing analytics turn event
without changing app-server protocol or rollout persistence.
- Use the existing turn `status` to distinguish completed, failed, and
interrupted profiles.

Exact sampling/tool overlap is intentionally omitted because measuring
tool completion accurately would require hooks in the tool execution
path.

## Validation

- Add app-server end-to-end coverage for a single-sampling turn with no
blocking tool work.
- Add app-server end-to-end coverage for `request_user_input` blocking
followed by a second sampling request.
- CI is running on the PR; tests were not executed locally per
repository guidance.
This commit is contained in:
Ahmed Ibrahim
2026-06-05 11:27:10 -07:00
committed by GitHub
Unverified
parent 82b15b65e2
commit 8d72fb6de9
11 changed files with 530 additions and 101 deletions
@@ -9,6 +9,7 @@ use app_test_support::create_final_assistant_message_sse_response;
use app_test_support::create_mock_responses_server_repeating_assistant;
use app_test_support::create_mock_responses_server_sequence;
use app_test_support::create_mock_responses_server_sequence_unchecked;
use app_test_support::create_request_user_input_sse_response;
use app_test_support::create_shell_command_sse_response;
use app_test_support::format_with_current_shell_display;
use app_test_support::to_response;
@@ -78,6 +79,7 @@ use std::collections::HashMap;
use std::path::Path;
use tempfile::TempDir;
use tokio::time::timeout;
use wiremock::ResponseTemplate;
use super::analytics::mount_analytics_capture;
use super::analytics::wait_for_analytics_event;
@@ -819,8 +821,20 @@ async fn thread_start_omits_empty_instruction_overrides_from_model_request() ->
#[tokio::test]
async fn turn_start_tracks_turn_event_analytics() -> Result<()> {
let responses = vec![create_final_assistant_message_sse_response("Done")?];
let server = create_mock_responses_server_sequence_unchecked(responses).await;
let server = responses::start_mock_server().await;
let response_mock = responses::mount_response_sequence(
&server,
vec![
ResponseTemplate::new(500).set_body_json(json!({
"error": {
"type": "server_error",
"message": "synthetic retryable error"
}
})),
responses::sse_response(create_final_assistant_message_sse_response("Done")?),
],
)
.await;
let codex_home = TempDir::new()?;
write_mock_responses_config_toml_with_chatgpt_base_url(
@@ -828,6 +842,10 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> {
&server.uri(),
&server.uri(),
)?;
let config_path = codex_home.path().join("config.toml");
let config = std::fs::read_to_string(&config_path)?
.replace("stream_max_retries = 0", "stream_max_retries = 1");
std::fs::write(config_path, config)?;
mount_analytics_capture(&server, codex_home.path()).await?;
let mut mcp = TestAppServer::new_without_managed_config(codex_home.path()).await?;
@@ -908,6 +926,136 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> {
assert_eq!(event["event_params"]["output_tokens"], 0);
assert_eq!(event["event_params"]["reasoning_output_tokens"], 0);
assert_eq!(event["event_params"]["total_tokens"], 0);
let params = &event["event_params"];
let timings_are_numbers = [
"before_first_sampling_ms",
"sampling_ms",
"between_sampling_overhead_ms",
"tool_blocking_ms",
"after_last_sampling_ms",
]
.into_iter()
.all(|field| params[field].as_u64().is_some());
assert_eq!(
json!({
"timingsAreNumbers": timings_are_numbers,
"toolBlockingMs": params["tool_blocking_ms"],
"samplingRequestCount": params["sampling_request_count"],
"samplingRetryCount": params["sampling_retry_count"],
"responseRequestCount": response_mock.requests().len(),
}),
json!({
"timingsAreNumbers": true,
"toolBlockingMs": 0,
"samplingRequestCount": 2,
"samplingRetryCount": 1,
"responseRequestCount": 2,
})
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn turn_profile_tracks_blocking_tool_and_follow_up_sampling() -> Result<()> {
let responses = vec![
create_request_user_input_sse_response("call1")?,
create_final_assistant_message_sse_response("Done")?,
];
let server = create_mock_responses_server_sequence(responses).await;
let codex_home = TempDir::new()?;
write_mock_responses_config_toml_with_chatgpt_base_url(
codex_home.path(),
&server.uri(),
&server.uri(),
)?;
mount_analytics_capture(&server, codex_home.path()).await?;
let mut mcp = TestAppServer::new_without_managed_config(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams {
model: Some("mock-model".to_string()),
..Default::default()
})
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
client_user_message_id: None,
input: vec![V2UserInput::Text {
text: "ask something".to_string(),
text_elements: Vec::new(),
}],
collaboration_mode: Some(CollaborationMode {
mode: ModeKind::Plan,
settings: Settings {
model: "mock-model".to_string(),
reasoning_effort: Some(ReasoningEffort::Medium),
developer_instructions: None,
},
}),
..Default::default()
})
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let server_req = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_request_message(),
)
.await??;
let ServerRequest::ToolRequestUserInput { request_id, .. } = server_req else {
panic!("expected ToolRequestUserInput request, got: {server_req:?}");
};
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
mcp.send_response(
request_id,
json!({
"answers": {
"confirm_path": { "answers": ["yes"] }
}
}),
)
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let event = wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_event").await?;
let params = &event["event_params"];
assert_eq!(
json!({
"toolBlockingIsPositive": params["tool_blocking_ms"]
.as_u64()
.is_some_and(|duration| duration > 0),
"samplingRequestCount": params["sampling_request_count"],
"samplingRetryCount": params["sampling_retry_count"],
"status": params["status"],
}),
json!({
"toolBlockingIsPositive": true,
"samplingRequestCount": 2,
"samplingRetryCount": 0,
"status": "completed",
})
);
Ok(())
}