mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
[rollout_trace] Record core session rollout traces (#18877)
## Summary Wires rollout trace recording into `codex-core` session and turn execution. This records the core model request/response, compaction, and session lifecycle boundaries needed for replay without yet tracing every nested runtime/tool boundary. ## Stack This is PR 2/5 in the rollout trace stack. - [#18876](https://github.com/openai/codex/pull/18876): Add rollout trace crate - [#18877](https://github.com/openai/codex/pull/18877): Record core session rollout traces - [#18878](https://github.com/openai/codex/pull/18878): Trace tool and code-mode boundaries - [#18879](https://github.com/openai/codex/pull/18879): Trace sessions and multi-agent edges - [#18880](https://github.com/openai/codex/pull/18880): Add debug trace reduction command ## Review Notes This layer is the first live integration point. The important review question is whether trace recording is isolated from normal session behavior: trace failures should not become user-visible execution failures, and recording should preserve the existing turn/session lifecycle semantics. The PR depends on the reducer/data model from the first stack entry and only introduces the core recorder surface that later PRs use for richer runtime and relationship events.
This commit is contained in:
committed by
GitHub
Unverified
parent
79ea577156
commit
f67383bcba
+69
-15
@@ -80,6 +80,9 @@ use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use codex_protocol::protocol::W3cTraceContext;
|
||||
use codex_rollout_trace::CompactionTraceContext;
|
||||
use codex_rollout_trace::InferenceTraceAttempt;
|
||||
use codex_rollout_trace::InferenceTraceContext;
|
||||
use codex_tools::create_tools_json_for_responses_api;
|
||||
use eventsource_stream::Event;
|
||||
use eventsource_stream::EventStreamError;
|
||||
@@ -411,6 +414,7 @@ impl ModelClient {
|
||||
effort: Option<ReasoningEffortConfig>,
|
||||
summary: ReasoningSummaryConfig,
|
||||
session_telemetry: &SessionTelemetry,
|
||||
compaction_trace: &CompactionTraceContext,
|
||||
) -> Result<Vec<ResponseItem>> {
|
||||
if prompt.input.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
@@ -469,10 +473,13 @@ impl ModelClient {
|
||||
extra_headers.extend(build_conversation_headers(Some(
|
||||
self.state.conversation_id.to_string(),
|
||||
)));
|
||||
client
|
||||
let trace_attempt = compaction_trace.start_attempt(&payload);
|
||||
let result = client
|
||||
.compact_input(&payload, extra_headers)
|
||||
.await
|
||||
.map_err(map_api_error)
|
||||
.map_err(map_api_error);
|
||||
trace_attempt.record_result(result.as_deref());
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) async fn create_realtime_call_with_headers(
|
||||
@@ -1156,6 +1163,7 @@ impl ModelClientSession {
|
||||
summary: ReasoningSummaryConfig,
|
||||
service_tier: Option<ServiceTier>,
|
||||
turn_metadata_header: Option<&str>,
|
||||
inference_trace: &InferenceTraceContext,
|
||||
) -> Result<ResponseStream> {
|
||||
if let Some(path) = &*CODEX_RS_SSE_FIXTURE {
|
||||
warn!(path, "Streaming from fixture");
|
||||
@@ -1164,7 +1172,11 @@ impl ModelClientSession {
|
||||
self.client.state.provider.info().stream_idle_timeout(),
|
||||
)
|
||||
.map_err(map_api_error)?;
|
||||
let (stream, _last_request_rx) = map_response_stream(stream, session_telemetry.clone());
|
||||
let (stream, _last_request_rx) = map_response_stream(
|
||||
stream,
|
||||
session_telemetry.clone(),
|
||||
InferenceTraceAttempt::disabled(),
|
||||
);
|
||||
return Ok(stream);
|
||||
}
|
||||
|
||||
@@ -1198,6 +1210,8 @@ impl ModelClientSession {
|
||||
summary,
|
||||
service_tier,
|
||||
)?;
|
||||
let inference_trace_attempt = inference_trace.start_attempt();
|
||||
inference_trace_attempt.record_started(&request);
|
||||
let client = ApiResponsesClient::new(
|
||||
transport,
|
||||
client_setup.api_provider,
|
||||
@@ -1208,12 +1222,17 @@ impl ModelClientSession {
|
||||
|
||||
match stream_result {
|
||||
Ok(stream) => {
|
||||
let (stream, _) = map_response_stream(stream, session_telemetry.clone());
|
||||
let (stream, _) = map_response_stream(
|
||||
stream,
|
||||
session_telemetry.clone(),
|
||||
inference_trace_attempt,
|
||||
);
|
||||
return Ok(stream);
|
||||
}
|
||||
Err(ApiError::Transport(
|
||||
unauthorized_transport @ TransportError::Http { status, .. },
|
||||
)) if status == StatusCode::UNAUTHORIZED => {
|
||||
inference_trace_attempt.record_failed(&unauthorized_transport);
|
||||
pending_retry = PendingUnauthorizedRetry::from_recovery(
|
||||
handle_unauthorized(
|
||||
unauthorized_transport,
|
||||
@@ -1224,7 +1243,11 @@ impl ModelClientSession {
|
||||
);
|
||||
continue;
|
||||
}
|
||||
Err(err) => return Err(map_api_error(err)),
|
||||
Err(err) => {
|
||||
let err = map_api_error(err);
|
||||
inference_trace_attempt.record_failed(&err);
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1255,6 +1278,7 @@ impl ModelClientSession {
|
||||
turn_metadata_header: Option<&str>,
|
||||
warmup: bool,
|
||||
request_trace: Option<W3cTraceContext>,
|
||||
inference_trace: &InferenceTraceContext,
|
||||
) -> Result<WebsocketStreamOutcome> {
|
||||
let auth_manager = self.client.state.provider.auth_manager();
|
||||
|
||||
@@ -1329,17 +1353,33 @@ impl ModelClientSession {
|
||||
|
||||
let ws_request = self.prepare_websocket_request(ws_payload, &request);
|
||||
self.websocket_session.last_request = Some(request);
|
||||
let stream_result = self.websocket_session.connection.as_ref().ok_or_else(|| {
|
||||
map_api_error(ApiError::Stream(
|
||||
"websocket connection is unavailable".to_string(),
|
||||
))
|
||||
})?;
|
||||
let stream_result = stream_result
|
||||
let inference_trace_attempt = if warmup {
|
||||
// Prewarm sends `generate=false`; it is connection setup, not a
|
||||
// model inference attempt that should appear in rollout traces.
|
||||
InferenceTraceAttempt::disabled()
|
||||
} else {
|
||||
inference_trace.start_attempt()
|
||||
};
|
||||
inference_trace_attempt.record_started(&ws_request);
|
||||
let websocket_connection =
|
||||
self.websocket_session.connection.as_ref().ok_or_else(|| {
|
||||
map_api_error(ApiError::Stream(
|
||||
"websocket connection is unavailable".to_string(),
|
||||
))
|
||||
})?;
|
||||
let stream_result = websocket_connection
|
||||
.stream_request(ws_request, self.websocket_session.connection_reused())
|
||||
.await
|
||||
.map_err(map_api_error)?;
|
||||
let (stream, last_request_rx) =
|
||||
map_response_stream(stream_result, session_telemetry.clone());
|
||||
.map_err(|err| {
|
||||
let err = map_api_error(err);
|
||||
inference_trace_attempt.record_failed(&err);
|
||||
err
|
||||
})?;
|
||||
let (stream, last_request_rx) = map_response_stream(
|
||||
stream_result,
|
||||
session_telemetry.clone(),
|
||||
inference_trace_attempt,
|
||||
);
|
||||
self.websocket_session.last_response_rx = Some(last_request_rx);
|
||||
return Ok(WebsocketStreamOutcome::Stream(stream));
|
||||
}
|
||||
@@ -1398,6 +1438,7 @@ impl ModelClientSession {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let disabled_trace = InferenceTraceContext::disabled();
|
||||
match self
|
||||
.stream_responses_websocket(
|
||||
prompt,
|
||||
@@ -1409,6 +1450,7 @@ impl ModelClientSession {
|
||||
turn_metadata_header,
|
||||
/*warmup*/ true,
|
||||
current_span_w3c_trace_context(),
|
||||
&disabled_trace,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1437,7 +1479,9 @@ impl ModelClientSession {
|
||||
/// The caller is responsible for passing per-turn settings explicitly (model selection,
|
||||
/// reasoning settings, telemetry context, and turn metadata). This method will prefer the
|
||||
/// Responses WebSocket transport when the provider supports it and it remains healthy, and will
|
||||
/// fall back to the HTTP Responses API transport otherwise.
|
||||
/// fall back to the HTTP Responses API transport otherwise. The trace context may be enabled or
|
||||
/// disabled, but is always explicit so transport paths do not need separate trace/no-trace
|
||||
/// branches.
|
||||
pub async fn stream(
|
||||
&mut self,
|
||||
prompt: &Prompt,
|
||||
@@ -1447,6 +1491,7 @@ impl ModelClientSession {
|
||||
summary: ReasoningSummaryConfig,
|
||||
service_tier: Option<ServiceTier>,
|
||||
turn_metadata_header: Option<&str>,
|
||||
inference_trace: &InferenceTraceContext,
|
||||
) -> Result<ResponseStream> {
|
||||
let wire_api = self.client.state.provider.info().wire_api;
|
||||
match wire_api {
|
||||
@@ -1464,6 +1509,7 @@ impl ModelClientSession {
|
||||
turn_metadata_header,
|
||||
/*warmup*/ false,
|
||||
request_trace,
|
||||
inference_trace,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -1482,6 +1528,7 @@ impl ModelClientSession {
|
||||
summary,
|
||||
service_tier,
|
||||
turn_metadata_header,
|
||||
inference_trace,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1577,6 +1624,7 @@ fn parent_thread_id_header_value(session_source: &SessionSource) -> Option<Strin
|
||||
fn map_response_stream<S>(
|
||||
api_stream: S,
|
||||
session_telemetry: SessionTelemetry,
|
||||
inference_trace_attempt: InferenceTraceAttempt,
|
||||
) -> (ResponseStream, oneshot::Receiver<LastResponse>)
|
||||
where
|
||||
S: futures::Stream<Item = std::result::Result<ResponseEvent, ApiError>>
|
||||
@@ -1617,6 +1665,11 @@ where
|
||||
usage.total_tokens,
|
||||
);
|
||||
}
|
||||
inference_trace_attempt.record_completed(
|
||||
&response_id,
|
||||
&token_usage,
|
||||
&items_added,
|
||||
);
|
||||
if let Some(sender) = tx_last_response.take() {
|
||||
let _ = sender.send(LastResponse {
|
||||
response_id: response_id.clone(),
|
||||
@@ -1641,6 +1694,7 @@ where
|
||||
}
|
||||
Err(err) => {
|
||||
let mapped = map_api_error(err);
|
||||
inference_trace_attempt.record_failed(&mapped);
|
||||
if !logged_error {
|
||||
session_telemetry.see_event_completed_failed(&mapped);
|
||||
logged_error = true;
|
||||
|
||||
Reference in New Issue
Block a user