mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
[codex] Expose service tier and reasoning effort in OTEL (#29155)
## Summary NVIDIA asked to measure Fast mode usage and reasoning effort from Codex CLI OTEL logs. Add the finalized `service_tier` and `model_reasoning_effort` to the existing `codex.sse_event` `response.completed` record. This intentionally reuses the existing completion event and leaves transport APIs and shared telemetry plumbing unchanged. ## Testing - `cargo build -p codex-cli --bin codex` - `just test -p codex-core responses_api_emits_api_request_event` - End-to-end with the built CLI and a local OTLP/HTTP collector: - Fast/high emitted `service_tier=priority` and `model_reasoning_effort=high` with token usage. - Standard/low omitted `service_tier` and emitted `model_reasoning_effort=low` with token usage.
This commit is contained in:
committed by
GitHub
Unverified
parent
27f22b54ae
commit
e0ac5d3c15
@@ -162,6 +162,19 @@ pub(crate) struct CompactConversationRequestSettings {
|
||||
pub(crate) service_tier: Option<String>,
|
||||
}
|
||||
|
||||
fn session_telemetry_for_request(
|
||||
session_telemetry: &SessionTelemetry,
|
||||
request: &ResponsesApiRequest,
|
||||
) -> SessionTelemetry {
|
||||
session_telemetry.clone().with_inference_request(
|
||||
request.service_tier.as_deref(),
|
||||
request
|
||||
.reasoning
|
||||
.as_ref()
|
||||
.and_then(|reasoning| reasoning.effort.as_ref()),
|
||||
)
|
||||
}
|
||||
|
||||
/// Session-scoped state shared by all [`ModelClient`] clones.
|
||||
///
|
||||
/// This is intentionally kept minimal so `ModelClient` does not need to hold a full `Config`. Most
|
||||
@@ -1319,6 +1332,8 @@ impl ModelClientSession {
|
||||
let store = request.store;
|
||||
self.client
|
||||
.prepare_response_items_for_request(&mut request.input, store);
|
||||
let request_session_telemetry =
|
||||
session_telemetry_for_request(session_telemetry, &request);
|
||||
let inference_trace_attempt = inference_trace.start_attempt();
|
||||
inference_trace_attempt.add_request_headers(&mut options.extra_headers);
|
||||
inference_trace_attempt.record_started(&request);
|
||||
@@ -1334,7 +1349,7 @@ impl ModelClientSession {
|
||||
Ok(stream) => {
|
||||
let (stream, _) = map_response_stream(
|
||||
stream,
|
||||
session_telemetry.clone(),
|
||||
request_session_telemetry,
|
||||
inference_trace_attempt,
|
||||
Arc::clone(&self.client.state.provider),
|
||||
);
|
||||
@@ -1426,6 +1441,12 @@ impl ModelClientSession {
|
||||
service_tier.clone(),
|
||||
responses_metadata,
|
||||
)?;
|
||||
let request_session_telemetry = if warmup {
|
||||
// `generate=false` prewarm is connection setup, not an inference request.
|
||||
session_telemetry.clone()
|
||||
} else {
|
||||
session_telemetry_for_request(session_telemetry, &request)
|
||||
};
|
||||
let mut client_metadata = self
|
||||
.client
|
||||
.build_ws_client_metadata(responses_metadata, model_info.use_responses_lite);
|
||||
@@ -1529,7 +1550,7 @@ impl ModelClientSession {
|
||||
})?;
|
||||
let (stream, last_request_rx) = map_response_stream(
|
||||
stream_result,
|
||||
session_telemetry.clone(),
|
||||
request_session_telemetry,
|
||||
inference_trace_attempt,
|
||||
Arc::clone(&self.client.state.provider),
|
||||
);
|
||||
|
||||
@@ -3,6 +3,7 @@ use codex_features::Feature;
|
||||
use codex_otel::SessionTelemetry;
|
||||
use codex_otel::TelemetryAuthMode;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::config_types::ServiceTier;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::openai_models::ReasoningEffort;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
@@ -114,9 +115,17 @@ fn extract_log_field_does_not_confuse_similar_keys() {
|
||||
async fn responses_api_emits_api_request_event() {
|
||||
let server = start_mock_server().await;
|
||||
|
||||
mount_sse_once(&server, sse(vec![ev_completed("done")])).await;
|
||||
let response_mock = mount_sse_once(&server, sse(vec![ev_completed("done")])).await;
|
||||
|
||||
let TestCodex { codex, .. } = test_codex().build(&server).await.unwrap();
|
||||
let TestCodex { codex, .. } = test_codex()
|
||||
.with_model("gpt-5.4")
|
||||
.with_config(|config| {
|
||||
config.service_tier = Some(ServiceTier::Fast.request_value().to_string());
|
||||
config.model_reasoning_effort = Some(ReasoningEffort::High);
|
||||
})
|
||||
.build(&server)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
codex
|
||||
.submit(Op::UserInput {
|
||||
@@ -134,6 +143,10 @@ async fn responses_api_emits_api_request_event() {
|
||||
|
||||
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
let request_body = response_mock.single_request().body_json();
|
||||
assert_eq!(request_body["service_tier"].as_str(), Some("priority"));
|
||||
assert_eq!(request_body["reasoning"]["effort"].as_str(), Some("high"));
|
||||
|
||||
logs_assert(|lines: &[&str]| {
|
||||
lines
|
||||
.iter()
|
||||
@@ -142,6 +155,21 @@ async fn responses_api_emits_api_request_event() {
|
||||
.unwrap_or_else(|| Err("expected codex.api_request event".to_string()))
|
||||
});
|
||||
|
||||
logs_assert(|lines: &[&str]| {
|
||||
lines
|
||||
.iter()
|
||||
.find(|line| {
|
||||
line.contains("codex.sse_event")
|
||||
&& line.contains("event.kind=response.completed")
|
||||
&& line.contains("service_tier=\"priority\"")
|
||||
&& line.contains("model_reasoning_effort=\"high\"")
|
||||
})
|
||||
.map(|_| Ok(()))
|
||||
.unwrap_or_else(|| {
|
||||
Err("expected response.completed event with inference attributes".to_string())
|
||||
})
|
||||
});
|
||||
|
||||
logs_assert(|lines: &[&str]| {
|
||||
lines
|
||||
.iter()
|
||||
|
||||
@@ -94,6 +94,8 @@ pub struct SessionTelemetryMetadata {
|
||||
pub(crate) session_source: String,
|
||||
pub(crate) model: String,
|
||||
pub(crate) slug: String,
|
||||
pub(crate) service_tier: Option<String>,
|
||||
pub(crate) model_reasoning_effort: Option<String>,
|
||||
pub(crate) log_user_prompts: bool,
|
||||
pub(crate) app_version: &'static str,
|
||||
pub(crate) terminal_type: String,
|
||||
@@ -118,6 +120,16 @@ impl SessionTelemetry {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_inference_request(
|
||||
mut self,
|
||||
service_tier: Option<&str>,
|
||||
model_reasoning_effort: Option<&ReasoningEffort>,
|
||||
) -> Self {
|
||||
self.metadata.service_tier = service_tier.map(str::to_owned);
|
||||
self.metadata.model_reasoning_effort = model_reasoning_effort.map(ToString::to_string);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_metrics_service_name(mut self, service_name: &str) -> Self {
|
||||
self.metadata.service_name = Some(sanitize_metric_tag_value(service_name));
|
||||
self
|
||||
@@ -389,6 +401,8 @@ impl SessionTelemetry {
|
||||
session_source: session_source.to_string(),
|
||||
model: model.to_owned(),
|
||||
slug: slug.to_owned(),
|
||||
service_tier: None,
|
||||
model_reasoning_effort: None,
|
||||
log_user_prompts,
|
||||
app_version: env!("CARGO_PKG_VERSION"),
|
||||
terminal_type,
|
||||
@@ -896,6 +910,8 @@ impl SessionTelemetry {
|
||||
cached_token_count = cached_token_count,
|
||||
reasoning_token_count = reasoning_token_count,
|
||||
tool_token_count = %tool_token_count,
|
||||
service_tier = self.metadata.service_tier.as_deref(),
|
||||
model_reasoning_effort = self.metadata.model_reasoning_effort.as_deref(),
|
||||
},
|
||||
log: {},
|
||||
trace: {},
|
||||
|
||||
Reference in New Issue
Block a user