mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
[codex] Add turn profiling analytics (#26484)
## Summary Add flat profiling fields to `codex_turn_event` so analytics can explain where turn wall-clock time is spent without changing tool execution behavior. The profile reports: - time before the first sampling request - sampling time across all attempts and follow-ups - overhead between sampling requests - time blocked in the post-sampling tool drain - time after the final sampling request - sampling request and retry counts ## Implementation - Extend the existing turn timing state with constant-memory phase accounting and one RAII phase guard. - Observe sampling and the existing post-sampling drain only at turn orchestration boundaries. - Keep tool runtime, tool futures, response item handling, and turn lifecycle values unchanged. - Add the profiling fields directly to the existing analytics turn event without changing app-server protocol or rollout persistence. - Use the existing turn `status` to distinguish completed, failed, and interrupted profiles. Exact sampling/tool overlap is intentionally omitted because measuring tool completion accurately would require hooks in the tool execution path. ## Validation - Add app-server end-to-end coverage for a single-sampling turn with no blocking tool work. - Add app-server end-to-end coverage for `request_user_input` blocking followed by a second sampling request. - CI is running on the PR; tests were not executed locally per repository guidance.
This commit is contained in:
committed by
GitHub
Unverified
parent
82b15b65e2
commit
8d72fb6de9
@@ -9,6 +9,7 @@ use app_test_support::create_final_assistant_message_sse_response;
|
||||
use app_test_support::create_mock_responses_server_repeating_assistant;
|
||||
use app_test_support::create_mock_responses_server_sequence;
|
||||
use app_test_support::create_mock_responses_server_sequence_unchecked;
|
||||
use app_test_support::create_request_user_input_sse_response;
|
||||
use app_test_support::create_shell_command_sse_response;
|
||||
use app_test_support::format_with_current_shell_display;
|
||||
use app_test_support::to_response;
|
||||
@@ -78,6 +79,7 @@ use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use tempfile::TempDir;
|
||||
use tokio::time::timeout;
|
||||
use wiremock::ResponseTemplate;
|
||||
|
||||
use super::analytics::mount_analytics_capture;
|
||||
use super::analytics::wait_for_analytics_event;
|
||||
@@ -819,8 +821,20 @@ async fn thread_start_omits_empty_instruction_overrides_from_model_request() ->
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_start_tracks_turn_event_analytics() -> Result<()> {
|
||||
let responses = vec![create_final_assistant_message_sse_response("Done")?];
|
||||
let server = create_mock_responses_server_sequence_unchecked(responses).await;
|
||||
let server = responses::start_mock_server().await;
|
||||
let response_mock = responses::mount_response_sequence(
|
||||
&server,
|
||||
vec![
|
||||
ResponseTemplate::new(500).set_body_json(json!({
|
||||
"error": {
|
||||
"type": "server_error",
|
||||
"message": "synthetic retryable error"
|
||||
}
|
||||
})),
|
||||
responses::sse_response(create_final_assistant_message_sse_response("Done")?),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
write_mock_responses_config_toml_with_chatgpt_base_url(
|
||||
@@ -828,6 +842,10 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> {
|
||||
&server.uri(),
|
||||
&server.uri(),
|
||||
)?;
|
||||
let config_path = codex_home.path().join("config.toml");
|
||||
let config = std::fs::read_to_string(&config_path)?
|
||||
.replace("stream_max_retries = 0", "stream_max_retries = 1");
|
||||
std::fs::write(config_path, config)?;
|
||||
mount_analytics_capture(&server, codex_home.path()).await?;
|
||||
|
||||
let mut mcp = TestAppServer::new_without_managed_config(codex_home.path()).await?;
|
||||
@@ -908,6 +926,136 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> {
|
||||
assert_eq!(event["event_params"]["output_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["reasoning_output_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["total_tokens"], 0);
|
||||
let params = &event["event_params"];
|
||||
let timings_are_numbers = [
|
||||
"before_first_sampling_ms",
|
||||
"sampling_ms",
|
||||
"between_sampling_overhead_ms",
|
||||
"tool_blocking_ms",
|
||||
"after_last_sampling_ms",
|
||||
]
|
||||
.into_iter()
|
||||
.all(|field| params[field].as_u64().is_some());
|
||||
assert_eq!(
|
||||
json!({
|
||||
"timingsAreNumbers": timings_are_numbers,
|
||||
"toolBlockingMs": params["tool_blocking_ms"],
|
||||
"samplingRequestCount": params["sampling_request_count"],
|
||||
"samplingRetryCount": params["sampling_retry_count"],
|
||||
"responseRequestCount": response_mock.requests().len(),
|
||||
}),
|
||||
json!({
|
||||
"timingsAreNumbers": true,
|
||||
"toolBlockingMs": 0,
|
||||
"samplingRequestCount": 2,
|
||||
"samplingRetryCount": 1,
|
||||
"responseRequestCount": 2,
|
||||
})
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn turn_profile_tracks_blocking_tool_and_follow_up_sampling() -> Result<()> {
|
||||
let responses = vec![
|
||||
create_request_user_input_sse_response("call1")?,
|
||||
create_final_assistant_message_sse_response("Done")?,
|
||||
];
|
||||
let server = create_mock_responses_server_sequence(responses).await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
write_mock_responses_config_toml_with_chatgpt_base_url(
|
||||
codex_home.path(),
|
||||
&server.uri(),
|
||||
&server.uri(),
|
||||
)?;
|
||||
mount_analytics_capture(&server, codex_home.path()).await?;
|
||||
|
||||
let mut mcp = TestAppServer::new_without_managed_config(codex_home.path()).await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
|
||||
let thread_req = mcp
|
||||
.send_thread_start_request(ThreadStartParams {
|
||||
model: Some("mock-model".to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let thread_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
|
||||
|
||||
let turn_req = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: thread.id.clone(),
|
||||
client_user_message_id: None,
|
||||
input: vec![V2UserInput::Text {
|
||||
text: "ask something".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
collaboration_mode: Some(CollaborationMode {
|
||||
mode: ModeKind::Plan,
|
||||
settings: Settings {
|
||||
model: "mock-model".to_string(),
|
||||
reasoning_effort: Some(ReasoningEffort::Medium),
|
||||
developer_instructions: None,
|
||||
},
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let server_req = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_request_message(),
|
||||
)
|
||||
.await??;
|
||||
let ServerRequest::ToolRequestUserInput { request_id, .. } = server_req else {
|
||||
panic!("expected ToolRequestUserInput request, got: {server_req:?}");
|
||||
};
|
||||
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
|
||||
mcp.send_response(
|
||||
request_id,
|
||||
json!({
|
||||
"answers": {
|
||||
"confirm_path": { "answers": ["yes"] }
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let event = wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_event").await?;
|
||||
let params = &event["event_params"];
|
||||
assert_eq!(
|
||||
json!({
|
||||
"toolBlockingIsPositive": params["tool_blocking_ms"]
|
||||
.as_u64()
|
||||
.is_some_and(|duration| duration > 0),
|
||||
"samplingRequestCount": params["sampling_request_count"],
|
||||
"samplingRetryCount": params["sampling_retry_count"],
|
||||
"status": params["status"],
|
||||
}),
|
||||
json!({
|
||||
"toolBlockingIsPositive": true,
|
||||
"samplingRequestCount": 2,
|
||||
"samplingRetryCount": 0,
|
||||
"status": "completed",
|
||||
})
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user