[codex] abort turns when rollout budgets expire (token budget 3/3) (#28707)

## Stack

Depends on #28494.

## Description

This PR propagates shared rollout-budget exhaustion through the existing
`CodexErr::TurnAborted` task result.

Each thread records its model usage against the same ledger. Once the
ledger is exhausted, that usage update and all later usage updates
return `TurnAborted`. The task wrapper emits the normal aborted-turn
event and lifecycle instead of completing the turn.

This is intentionally a soft boundary: there is no cross-thread
`Op::Interrupt` fanout. An in-flight thread can finish its current
response before it observes the exhausted ledger, but every thread
aborts at its next usage-accounting boundary.

## Tests

The integration coverage verifies that:

- the response that exhausts the budget aborts its turn;
- a later response also aborts because the shared ledger remains
exhausted; and
- sub-agent usage draws from the same shared ledger; and
- local and remote-v2 compaction abort without retrying or emitting a
generic error.

Local checks:

- `just test -p codex-core
exhausted_budget_aborts_current_and_later_turns`
- `just test -p codex-core subagent_usage_draws_from_the_shared_budget`
- `just test -p codex-core
abort_regular_task_emits_marker_before_turn_aborted`
- `just test -p codex-core
compaction_budget_exhaustion_aborts_without_error_or_retry`
- `just fix -p codex-core`
- `just fmt`
- `git diff --check`

The full workspace test suite was not run locally.
This commit is contained in:
rka-oai
2026-06-19 02:00:01 -07:00
committed by GitHub
Unverified
parent 7abfcf220b
commit dac588f413
13 changed files with 266 additions and 69 deletions
+8 -3
View File
@@ -2,10 +2,12 @@ use std::sync::Arc;
use super::SessionTask;
use super::SessionTaskContext;
use super::SessionTaskResult;
use super::emit_compact_metric;
use crate::session::TurnInput;
use crate::session::turn_context::TurnContext;
use crate::state::TaskKind;
use codex_protocol::error::CodexErr;
use codex_protocol::user_input::UserInput;
use tokio_util::sync::CancellationToken;
@@ -27,9 +29,9 @@ impl SessionTask for CompactTask {
ctx: Arc<TurnContext>,
_input: Vec<TurnInput>,
_cancellation_token: CancellationToken,
) -> Option<String> {
) -> SessionTaskResult {
let session = session.clone_session();
let _ = if crate::compact::should_use_remote_compact_task(ctx.provider.info()) {
let result = if crate::compact::should_use_remote_compact_task(ctx.provider.info()) {
if ctx
.config
.features
@@ -67,6 +69,9 @@ impl SessionTask for CompactTask {
}];
crate::compact::run_compact_task(session.clone(), ctx, input).await
};
None
if let Err(err @ CodexErr::TurnAborted) = result {
return Err(err);
}
Ok(None)
}
}
+46 -21
View File
@@ -54,6 +54,8 @@ use codex_protocol::protocol::TurnCompleteEvent;
use codex_protocol::protocol::WarningEvent;
use codex_features::Feature;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::models::ContentItem;
pub(crate) use compact::CompactTask;
pub(crate) use regular::RegularTask;
@@ -65,6 +67,8 @@ pub(crate) use user_shell::execute_user_shell_command;
const GRACEFULL_INTERRUPTION_TIMEOUT_MS: u64 = 100;
const TASK_COMPACT_METRIC: &str = "codex.task.compact";
pub(crate) type SessionTaskResult = CodexResult<Option<String>>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum InterruptedTurnHistoryMarker {
Disabled,
@@ -222,14 +226,16 @@ pub(crate) trait SessionTask: Send + Sync + 'static {
/// provided `cancellation_token` is cancelled when the session requests an
/// abort; implementers should watch for it and terminate quickly once it
/// fires. Returning [`Some`] yields a final message that
/// [`Session::on_task_finished`] will emit to the client.
/// [`Session::on_task_finished`] will emit to the client. Returning
/// [`CodexErr::TurnAborted`] completes the task through the aborted-turn
/// lifecycle instead.
fn run(
self: Arc<Self>,
session: Arc<SessionTaskContext>,
ctx: Arc<TurnContext>,
input: Vec<TurnInput>,
cancellation_token: CancellationToken,
) -> impl std::future::Future<Output = Option<String>> + Send;
) -> impl std::future::Future<Output = SessionTaskResult> + Send;
/// Gives the task a chance to perform cleanup after an abort.
///
@@ -258,7 +264,7 @@ pub(crate) trait AnySessionTask: Send + Sync + 'static {
ctx: Arc<TurnContext>,
input: Vec<TurnInput>,
cancellation_token: CancellationToken,
) -> BoxFuture<'static, Option<String>>;
) -> BoxFuture<'static, SessionTaskResult>;
fn abort<'a>(
&'a self,
@@ -285,7 +291,7 @@ where
ctx: Arc<TurnContext>,
input: Vec<TurnInput>,
cancellation_token: CancellationToken,
) -> BoxFuture<'static, Option<String>> {
) -> BoxFuture<'static, SessionTaskResult> {
Box::pin(SessionTask::run(
self,
session,
@@ -395,7 +401,7 @@ impl Session {
let handle = tokio::spawn(
async move {
let ctx_for_finish = Arc::clone(&ctx);
let last_agent_message = task_for_run
let task_result = task_for_run
.run(
Arc::clone(&session_ctx),
ctx,
@@ -418,8 +424,8 @@ impl Session {
.await;
}
if !task_cancellation_token.is_cancelled() {
// Emit completion uniformly from spawn site so all tasks share the same lifecycle.
sess.on_task_finished(Arc::clone(&ctx_for_finish), last_agent_message)
// Finish uniformly from the spawn site so all tasks share the same lifecycle.
sess.on_task_finished(Arc::clone(&ctx_for_finish), task_result)
.await;
}
done_clone.notify_waiters();
@@ -557,8 +563,16 @@ impl Session {
pub async fn on_task_finished(
self: &Arc<Self>,
turn_context: Arc<TurnContext>,
last_agent_message: Option<String>,
task_result: SessionTaskResult,
) {
let (last_agent_message, abort_reason) = match task_result {
Ok(last_agent_message) => (last_agent_message, None),
Err(CodexErr::TurnAborted) => (None, Some(TurnAbortReason::Interrupted)),
Err(err) => {
warn!(%err, "session task returned an unexpected error");
(None, None)
}
};
turn_context
.turn_metadata_state
.cancel_git_enrichment_task();
@@ -730,25 +744,36 @@ impl Session {
.turn_timing_state
.completed_at_and_duration_ms()
.await;
let time_to_first_token_ms = turn_context
.turn_timing_state
.time_to_first_token_ms()
.await;
self.services
.analytics_events_client
.track_turn_profile(TurnProfileFact {
turn_id: turn_context.sub_id.clone(),
profile: turn_context.turn_timing_state.complete_profile(),
});
self.emit_turn_stop_lifecycle(turn_context.extension_data.as_ref())
.await;
let event = EventMsg::TurnComplete(TurnCompleteEvent {
turn_id: turn_context.sub_id.clone(),
last_agent_message,
completed_at,
duration_ms,
time_to_first_token_ms,
});
let event = if let Some(reason) = abort_reason {
self.emit_turn_abort_lifecycle(reason.clone(), turn_context.extension_data.as_ref())
.await;
EventMsg::TurnAborted(TurnAbortedEvent {
turn_id: Some(turn_context.sub_id.clone()),
reason,
completed_at,
duration_ms,
})
} else {
let time_to_first_token_ms = turn_context
.turn_timing_state
.time_to_first_token_ms()
.await;
self.emit_turn_stop_lifecycle(turn_context.extension_data.as_ref())
.await;
EventMsg::TurnComplete(TurnCompleteEvent {
turn_id: turn_context.sub_id.clone(),
last_agent_message,
completed_at,
duration_ms,
time_to_first_token_ms,
})
};
self.send_event(turn_context.as_ref(), event).await;
self.services
.guardian_rejection_circuit_breaker
+5 -4
View File
@@ -14,6 +14,7 @@ use tracing::trace_span;
use super::SessionTask;
use super::SessionTaskContext;
use super::SessionTaskResult;
#[derive(Default)]
pub(crate) struct RegularTask;
@@ -39,7 +40,7 @@ impl SessionTask for RegularTask {
ctx: Arc<TurnContext>,
input: Vec<TurnInput>,
cancellation_token: CancellationToken,
) -> Option<String> {
) -> SessionTaskResult {
let sess = session.clone_session();
let turn_extension_data = session.turn_extension_data();
let run_turn_span = trace_span!("run_turn");
@@ -61,7 +62,7 @@ impl SessionTask for RegularTask {
.instrument(trace_span!("regular_task.prepare_run_turn"))
.await;
let prewarmed_client_session = match prewarmed_client_session {
SessionStartupPrewarmResolution::Cancelled => return None,
SessionStartupPrewarmResolution::Cancelled => return Ok(None),
SessionStartupPrewarmResolution::Unavailable { .. } => None,
SessionStartupPrewarmResolution::Ready(prewarmed_client_session) => {
Some(*prewarmed_client_session)
@@ -79,9 +80,9 @@ impl SessionTask for RegularTask {
cancellation_token.child_token(),
)
.instrument(run_turn_span.clone())
.await;
.await?;
if !sess.input_queue.has_pending_input(&sess.active_turn).await {
return last_agent_message;
return Ok(last_agent_message);
}
next_input = Vec::new();
}
+3 -2
View File
@@ -29,6 +29,7 @@ use codex_protocol::user_input::UserInput;
use super::SessionTask;
use super::SessionTaskContext;
use super::SessionTaskResult;
#[derive(Clone, Copy)]
pub(crate) struct ReviewTask;
@@ -54,7 +55,7 @@ impl SessionTask for ReviewTask {
ctx: Arc<TurnContext>,
input: Vec<TurnInput>,
cancellation_token: CancellationToken,
) -> Option<String> {
) -> SessionTaskResult {
session.session.services.session_telemetry.counter(
"codex.task.review",
/*inc*/ 1,
@@ -84,7 +85,7 @@ impl SessionTask for ReviewTask {
if !cancellation_token.is_cancelled() {
exit_review_mode(session.clone_session(), output.clone(), ctx.clone()).await;
}
None
Ok(None)
}
async fn abort(&self, session: Arc<SessionTaskContext>, ctx: Arc<TurnContext>) {
+3 -2
View File
@@ -41,6 +41,7 @@ use codex_shell_command::parse_command::parse_command;
use super::SessionTask;
use super::SessionTaskContext;
use super::SessionTaskResult;
use crate::session::session::Session;
use codex_protocol::models::PermissionProfile;
@@ -82,7 +83,7 @@ impl SessionTask for UserShellCommandTask {
turn_context: Arc<TurnContext>,
_input: Vec<TurnInput>,
cancellation_token: CancellationToken,
) -> Option<String> {
) -> SessionTaskResult {
execute_user_shell_command(
session.clone_session(),
turn_context,
@@ -91,7 +92,7 @@ impl SessionTask for UserShellCommandTask {
UserShellCommandMode::StandaloneTurn,
)
.await;
None
Ok(None)
}
}