mod compact; mod ghost_snapshot; mod regular; mod review; mod undo; mod user_shell; use std::sync::Arc; use std::time::Duration; use std::time::Instant; use futures::future::BoxFuture; use tokio::select; use tokio::sync::Notify; use tokio_util::sync::CancellationToken; use tokio_util::task::AbortOnDropHandle; use tracing::Instrument; use tracing::info_span; use tracing::trace; use tracing::warn; use crate::codex::Session; use crate::codex::TurnContext; use crate::contextual_user_message::TURN_ABORTED_CLOSE_TAG; use crate::contextual_user_message::TURN_ABORTED_OPEN_TAG; use crate::hook_runtime::PendingInputHookDisposition; use crate::hook_runtime::inspect_pending_input; use crate::hook_runtime::record_additional_contexts; use crate::hook_runtime::record_pending_input; use crate::state::ActiveTurn; use crate::state::RunningTask; use crate::state::TaskKind; use codex_login::AuthManager; use codex_models_manager::manager::ModelsManager; use codex_otel::SessionTelemetry; use codex_otel::metrics::names::TURN_E2E_DURATION_METRIC; use codex_otel::metrics::names::TURN_NETWORK_PROXY_METRIC; use codex_otel::metrics::names::TURN_TOKEN_USAGE_METRIC; use codex_otel::metrics::names::TURN_TOOL_CALL_METRIC; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::TurnAbortedEvent; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::user_input::UserInput; use codex_features::Feature; pub(crate) use compact::CompactTask; pub(crate) use ghost_snapshot::GhostSnapshotTask; pub(crate) use regular::RegularTask; pub(crate) use review::ReviewTask; pub(crate) use undo::UndoTask; pub(crate) use user_shell::UserShellCommandMode; pub(crate) use user_shell::UserShellCommandTask; pub(crate) use user_shell::execute_user_shell_command; const GRACEFULL_INTERRUPTION_TIMEOUT_MS: u64 = 100; const TURN_ABORTED_INTERRUPTED_GUIDANCE: &str = "The user interrupted the previous turn on purpose. Any running unified exec processes may still be running in the background. If any tools/commands were aborted, they may have partially executed."; /// Shared model-visible marker used by both the real interrupt path and /// interrupted fork snapshots. pub(crate) fn interrupted_turn_history_marker() -> ResponseItem { ResponseItem::Message { id: None, role: "user".to_string(), content: vec![ContentItem::InputText { text: format!( "{TURN_ABORTED_OPEN_TAG}\n{TURN_ABORTED_INTERRUPTED_GUIDANCE}\n{TURN_ABORTED_CLOSE_TAG}" ), }], end_turn: None, phase: None, } } fn emit_turn_network_proxy_metric( session_telemetry: &SessionTelemetry, network_proxy_active: bool, tmp_mem: (&str, &str), ) { let active = if network_proxy_active { "true" } else { "false" }; session_telemetry.counter( TURN_NETWORK_PROXY_METRIC, /*inc*/ 1, &[("active", active), tmp_mem], ); } /// Thin wrapper that exposes the parts of [`Session`] task runners need. #[derive(Clone)] pub(crate) struct SessionTaskContext { session: Arc, } impl SessionTaskContext { pub(crate) fn new(session: Arc) -> Self { Self { session } } pub(crate) fn clone_session(&self) -> Arc { Arc::clone(&self.session) } pub(crate) fn auth_manager(&self) -> Arc { Arc::clone(&self.session.services.auth_manager) } pub(crate) fn models_manager(&self) -> Arc { Arc::clone(&self.session.services.models_manager) } } /// Async task that drives a [`Session`] turn. /// /// Implementations encapsulate a specific Codex workflow (regular chat, /// reviews, ghost snapshots, etc.). Each task instance is owned by a /// [`Session`] and executed on a background Tokio task. The trait is /// intentionally small: implementers identify themselves via /// [`SessionTask::kind`], perform their work in [`SessionTask::run`], and may /// release resources in [`SessionTask::abort`]. pub(crate) trait SessionTask: Send + Sync + 'static { /// Describes the type of work the task performs so the session can /// surface it in telemetry and UI. fn kind(&self) -> TaskKind; /// Returns the tracing name for a spawned task span. fn span_name(&self) -> &'static str; /// Executes the task until completion or cancellation. /// /// Implementations typically stream protocol events using `session` and /// `ctx`, returning an optional final agent message when finished. The /// provided `cancellation_token` is cancelled when the session requests an /// abort; implementers should watch for it and terminate quickly once it /// fires. Returning [`Some`] yields a final message that /// [`Session::on_task_finished`] will emit to the client. fn run( self: Arc, session: Arc, ctx: Arc, input: Vec, cancellation_token: CancellationToken, ) -> impl std::future::Future> + Send; /// Gives the task a chance to perform cleanup after an abort. /// /// The default implementation is a no-op; override this if additional /// teardown or notifications are required once /// [`Session::abort_all_tasks`] cancels the task. fn abort( &self, session: Arc, ctx: Arc, ) -> impl std::future::Future + Send { async move { let _ = (session, ctx); } } } pub(crate) trait AnySessionTask: Send + Sync + 'static { fn kind(&self) -> TaskKind; fn span_name(&self) -> &'static str; fn run( self: Arc, session: Arc, ctx: Arc, input: Vec, cancellation_token: CancellationToken, ) -> BoxFuture<'static, Option>; fn abort<'a>( &'a self, session: Arc, ctx: Arc, ) -> BoxFuture<'a, ()>; } impl AnySessionTask for T where T: SessionTask, { fn kind(&self) -> TaskKind { SessionTask::kind(self) } fn span_name(&self) -> &'static str { SessionTask::span_name(self) } fn run( self: Arc, session: Arc, ctx: Arc, input: Vec, cancellation_token: CancellationToken, ) -> BoxFuture<'static, Option> { Box::pin(SessionTask::run( self, session, ctx, input, cancellation_token, )) } fn abort<'a>( &'a self, session: Arc, ctx: Arc, ) -> BoxFuture<'a, ()> { Box::pin(SessionTask::abort(self, session, ctx)) } } impl Session { pub async fn spawn_task( self: &Arc, turn_context: Arc, input: Vec, task: T, ) { self.abort_all_tasks(TurnAbortReason::Replaced).await; self.clear_connector_selection().await; self.start_task(turn_context, input, task).await; } async fn start_task( self: &Arc, turn_context: Arc, input: Vec, task: T, ) { let task: Arc = Arc::new(task); let task_kind = task.kind(); let span_name = task.span_name(); let started_at = Instant::now(); turn_context .turn_timing_state .mark_turn_started(started_at) .await; let token_usage_at_turn_start = self.total_token_usage().await.unwrap_or_default(); let cancellation_token = CancellationToken::new(); let done = Arc::new(Notify::new()); let queued_response_items = self.take_queued_response_items_for_next_turn().await; let mailbox_items = self.get_pending_input().await; let turn_state = { let mut active = self.active_turn.lock().await; let turn = active.get_or_insert_with(ActiveTurn::default); debug_assert!(turn.tasks.is_empty()); Arc::clone(&turn.turn_state) }; { let mut turn_state = turn_state.lock().await; turn_state.token_usage_at_turn_start = token_usage_at_turn_start; for item in queued_response_items { turn_state.push_pending_input(item); } for item in mailbox_items { turn_state.push_pending_input(item); } } let mut active = self.active_turn.lock().await; let turn = active.get_or_insert_with(ActiveTurn::default); debug_assert!(turn.tasks.is_empty()); let done_clone = Arc::clone(&done); let session_ctx = Arc::new(SessionTaskContext::new(Arc::clone(self))); let ctx = Arc::clone(&turn_context); let task_for_run = Arc::clone(&task); let task_cancellation_token = cancellation_token.child_token(); // Task-owned turn spans keep a core-owned span open for the // full task lifecycle after the submission dispatch span ends. let task_span = info_span!( "turn", otel.name = span_name, thread.id = %self.conversation_id, turn.id = %turn_context.sub_id, model = %turn_context.model_info.slug, ); let handle = tokio::spawn( async move { let ctx_for_finish = Arc::clone(&ctx); let last_agent_message = task_for_run .run( Arc::clone(&session_ctx), ctx, input, task_cancellation_token.child_token(), ) .await; let sess = session_ctx.clone_session(); sess.flush_rollout().await; if !task_cancellation_token.is_cancelled() { // Emit completion uniformly from spawn site so all tasks share the same lifecycle. sess.on_task_finished(Arc::clone(&ctx_for_finish), last_agent_message) .await; } done_clone.notify_waiters(); } .instrument(task_span), ); let timer = turn_context .session_telemetry .start_timer(TURN_E2E_DURATION_METRIC, &[]) .ok(); let running_task = RunningTask { done, handle: Arc::new(AbortOnDropHandle::new(handle)), kind: task_kind, task, cancellation_token, turn_context: Arc::clone(&turn_context), _timer: timer, }; turn.add_task(running_task); } /// Starts a regular turn when the session is idle and pending work is waiting. /// /// Pending work currently includes queued next-turn items and mailbox mail marked with /// `trigger_turn`. /// /// This helper generates a fresh sub-id for the synthetic turn before delegating to the /// explicit-sub-id variant. pub(crate) async fn maybe_start_turn_for_pending_work(self: &Arc) { self.maybe_start_turn_for_pending_work_with_sub_id(uuid::Uuid::new_v4().to_string()) .await; } /// Starts a regular turn with the provided sub-id when pending work should wake an idle /// session. /// /// The turn is created only when there are queued next-turn items or mailbox mail marked with /// `trigger_turn`, and only if the session is currently idle. pub(crate) async fn maybe_start_turn_for_pending_work_with_sub_id( self: &Arc, sub_id: String, ) { if !self.has_queued_response_items_for_next_turn().await && !self.has_trigger_turn_mailbox_items().await { return; } { let mut active_turn = self.active_turn.lock().await; if active_turn.is_some() { return; } *active_turn = Some(ActiveTurn::default()); } let turn_context = self.new_default_turn_with_sub_id(sub_id).await; self.maybe_emit_unknown_model_warning_for_turn(turn_context.as_ref()) .await; self.start_task(turn_context, Vec::new(), RegularTask::new()) .await; } pub async fn abort_all_tasks(self: &Arc, reason: TurnAbortReason) { if let Some(mut active_turn) = self.take_active_turn().await { for task in active_turn.drain_tasks() { self.handle_task_abort(task, reason.clone()).await; } // Let interrupted tasks observe cancellation before dropping pending approvals, or an // in-flight approval wait can surface as a model-visible rejection before TurnAborted. active_turn.clear_pending().await; } if reason == TurnAbortReason::Interrupted { self.maybe_start_turn_for_pending_work().await; } } pub async fn on_task_finished( self: &Arc, turn_context: Arc, last_agent_message: Option, ) { turn_context .turn_metadata_state .cancel_git_enrichment_task(); let mut pending_input = Vec::::new(); let mut should_clear_active_turn = false; let mut token_usage_at_turn_start = None; let mut turn_tool_calls = 0_u64; let turn_state = { let mut active = self.active_turn.lock().await; if let Some(at) = active.as_mut() && at.remove_task(&turn_context.sub_id) { should_clear_active_turn = true; let turn_state = Arc::clone(&at.turn_state); if should_clear_active_turn { *active = None; } Some(turn_state) } else { None } }; if let Some(turn_state) = turn_state { let mut ts = turn_state.lock().await; pending_input = ts.take_pending_input(); turn_tool_calls = ts.tool_calls; token_usage_at_turn_start = Some(ts.token_usage_at_turn_start.clone()); } if !pending_input.is_empty() { for pending_input_item in pending_input { match inspect_pending_input(self, &turn_context, pending_input_item).await { PendingInputHookDisposition::Accepted(pending_input) => { record_pending_input(self, &turn_context, *pending_input).await; } PendingInputHookDisposition::Blocked { additional_contexts, } => { record_additional_contexts(self, &turn_context, additional_contexts).await; } } } } // Emit token usage metrics. if let Some(token_usage_at_turn_start) = token_usage_at_turn_start { // TODO(jif): drop this let tmp_mem = ( "tmp_mem_enabled", if self.enabled(Feature::MemoryTool) { "true" } else { "false" }, ); let network_proxy_active = match self.services.network_proxy.as_ref() { Some(started_network_proxy) => { match started_network_proxy.proxy().current_cfg().await { Ok(config) => config.network.enabled, Err(err) => { warn!( "failed to read managed network proxy state for turn metrics: {err:#}" ); false } } } None => false, }; emit_turn_network_proxy_metric( &self.services.session_telemetry, network_proxy_active, tmp_mem, ); self.services.session_telemetry.histogram( TURN_TOOL_CALL_METRIC, i64::try_from(turn_tool_calls).unwrap_or(i64::MAX), &[tmp_mem], ); let total_token_usage = self.total_token_usage().await.unwrap_or_default(); let turn_token_usage = TokenUsage { input_tokens: (total_token_usage.input_tokens - token_usage_at_turn_start.input_tokens) .max(0), cached_input_tokens: (total_token_usage.cached_input_tokens - token_usage_at_turn_start.cached_input_tokens) .max(0), output_tokens: (total_token_usage.output_tokens - token_usage_at_turn_start.output_tokens) .max(0), reasoning_output_tokens: (total_token_usage.reasoning_output_tokens - token_usage_at_turn_start.reasoning_output_tokens) .max(0), total_tokens: (total_token_usage.total_tokens - token_usage_at_turn_start.total_tokens) .max(0), }; self.services.session_telemetry.histogram( TURN_TOKEN_USAGE_METRIC, turn_token_usage.total_tokens, &[("token_type", "total"), tmp_mem], ); self.services.session_telemetry.histogram( TURN_TOKEN_USAGE_METRIC, turn_token_usage.input_tokens, &[("token_type", "input"), tmp_mem], ); self.services.session_telemetry.histogram( TURN_TOKEN_USAGE_METRIC, turn_token_usage.cached_input(), &[("token_type", "cached_input"), tmp_mem], ); self.services.session_telemetry.histogram( TURN_TOKEN_USAGE_METRIC, turn_token_usage.output_tokens, &[("token_type", "output"), tmp_mem], ); self.services.session_telemetry.histogram( TURN_TOKEN_USAGE_METRIC, turn_token_usage.reasoning_output_tokens, &[("token_type", "reasoning_output"), tmp_mem], ); } let event = EventMsg::TurnComplete(TurnCompleteEvent { turn_id: turn_context.sub_id.clone(), last_agent_message, }); self.send_event(turn_context.as_ref(), event).await; if should_clear_active_turn { let session = Arc::clone(self); let _scheduler = tokio::task::spawn_blocking(move || { tokio::runtime::Handle::current().block_on(async move { session.maybe_start_turn_for_pending_work().await; }); }); } } async fn take_active_turn(&self) -> Option { let mut active = self.active_turn.lock().await; active.take() } pub(crate) async fn close_unified_exec_processes(&self) { self.services .unified_exec_manager .terminate_all_processes() .await; } pub(crate) async fn cleanup_after_interrupt(&self, turn_context: &Arc) { if let Some(manager) = turn_context.js_repl.manager_if_initialized() && let Err(err) = manager.interrupt_turn_exec(&turn_context.sub_id).await { warn!("failed to interrupt js_repl kernel: {err}"); } } async fn handle_task_abort(self: &Arc, task: RunningTask, reason: TurnAbortReason) { let sub_id = task.turn_context.sub_id.clone(); if task.cancellation_token.is_cancelled() { return; } trace!(task_kind = ?task.kind, sub_id, "aborting running task"); task.cancellation_token.cancel(); task.turn_context .turn_metadata_state .cancel_git_enrichment_task(); let session_task = task.task; select! { _ = task.done.notified() => { }, _ = tokio::time::sleep(Duration::from_millis(GRACEFULL_INTERRUPTION_TIMEOUT_MS)) => { warn!("task {sub_id} didn't complete gracefully after {}ms", GRACEFULL_INTERRUPTION_TIMEOUT_MS); } } task.handle.abort(); let session_ctx = Arc::new(SessionTaskContext::new(Arc::clone(self))); session_task .abort(session_ctx, Arc::clone(&task.turn_context)) .await; if reason == TurnAbortReason::Interrupted { self.cleanup_after_interrupt(&task.turn_context).await; let marker = interrupted_turn_history_marker(); self.record_into_history(std::slice::from_ref(&marker), task.turn_context.as_ref()) .await; self.persist_rollout_items(&[RolloutItem::ResponseItem(marker)]) .await; // Ensure the marker is durably visible before emitting TurnAborted: some clients // synchronously re-read the rollout on receipt of the abort event. self.flush_rollout().await; } let event = EventMsg::TurnAborted(TurnAbortedEvent { turn_id: Some(task.turn_context.sub_id.clone()), reason, }); self.send_event(task.turn_context.as_ref(), event).await; } } #[cfg(test)] #[path = "mod_tests.rs"] mod tests;