[codex] abort turns when rollout budgets expire (token budget 3/3) (#28707)

## Stack

Depends on #28494.

## Description

This PR propagates shared rollout-budget exhaustion through the existing
`CodexErr::TurnAborted` task result.

Each thread records its model usage against the same ledger. Once the
ledger is exhausted, that usage update and all later usage updates
return `TurnAborted`. The task wrapper emits the normal aborted-turn
event and lifecycle instead of completing the turn.

This is intentionally a soft boundary: there is no cross-thread
`Op::Interrupt` fanout. An in-flight thread can finish its current
response before it observes the exhausted ledger, but every thread
aborts at its next usage-accounting boundary.

## Tests

The integration coverage verifies that:

- the response that exhausts the budget aborts its turn;
- a later response also aborts because the shared ledger remains
exhausted; and
- sub-agent usage draws from the same shared ledger; and
- local and remote-v2 compaction abort without retrying or emitting a
generic error.

Local checks:

- `just test -p codex-core
exhausted_budget_aborts_current_and_later_turns`
- `just test -p codex-core subagent_usage_draws_from_the_shared_budget`
- `just test -p codex-core
abort_regular_task_emits_marker_before_turn_aborted`
- `just test -p codex-core
compaction_budget_exhaustion_aborts_without_error_or_retry`
- `just fix -p codex-core`
- `just fmt`
- `git diff --check`

The full workspace test suite was not run locally.
This commit is contained in:
rka-oai
2026-06-19 02:00:01 -07:00
committed by GitHub
Unverified
parent 7abfcf220b
commit dac588f413
13 changed files with 266 additions and 69 deletions
+8 -4
View File
@@ -3410,17 +3410,19 @@ impl Session {
&self,
turn_context: &TurnContext,
token_usage: Option<&TokenUsage>,
) {
self.record_token_usage_info(turn_context, token_usage)
) -> CodexResult<()> {
let result = self
.record_token_usage_info(turn_context, token_usage)
.await;
self.send_token_count_event(turn_context).await;
result
}
pub(crate) async fn record_token_usage_info(
&self,
turn_context: &TurnContext,
token_usage: Option<&TokenUsage>,
) {
) -> CodexResult<()> {
if let Some(token_usage) = token_usage {
let token_info = {
let mut state = self.state.lock().await;
@@ -3434,7 +3436,7 @@ impl Session {
}
state.token_info()
};
self.record_rollout_budget_usage(token_usage);
let budget_result = self.record_rollout_budget_usage(token_usage);
if let Some(token_info) = token_info.as_ref() {
for contributor in self.services.extensions.token_usage_contributors() {
contributor
@@ -3447,7 +3449,9 @@ impl Session {
.await;
}
}
budget_result?;
}
Ok(())
}
pub(crate) async fn recompute_token_usage(&self, turn_context: &TurnContext) {
+10 -3
View File
@@ -1,6 +1,8 @@
use super::session::Session;
use super::turn_context::TurnContext;
use crate::context::ContextualUserFragment;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::protocol::TokenUsage;
pub(super) async fn maybe_record_reminder(
@@ -21,10 +23,15 @@ pub(super) async fn maybe_record_reminder(
}
impl Session {
pub(crate) fn record_rollout_budget_usage(&self, usage: &TokenUsage) {
self.services
pub(crate) fn record_rollout_budget_usage(&self, usage: &TokenUsage) -> CodexResult<()> {
if self
.services
.agent_control
.rollout_budget()
.record_usage(usage);
.record_usage(usage)
{
return Err(CodexErr::TurnAborted);
}
Ok(())
}
}
+14 -11
View File
@@ -70,6 +70,7 @@ use crate::state::ActiveTurn;
use crate::state::TaskKind;
use crate::tasks::SessionTask;
use crate::tasks::SessionTaskContext;
use crate::tasks::SessionTaskResult;
use crate::tasks::UserShellCommandMode;
use crate::tasks::execute_user_shell_command;
use crate::tools::ToolRouter;
@@ -2150,10 +2151,12 @@ async fn record_token_usage_info_notifies_extension_contributors() {
session
.record_token_usage_info(&turn_context, Some(&first_usage))
.await;
.await
.expect("first usage should be recorded");
session
.record_token_usage_info(&turn_context, Some(&second_usage))
.await;
.await
.expect("second usage should be recorded");
let mut expected_total_usage = first_usage.clone();
expected_total_usage.add_assign(&second_usage);
@@ -6474,13 +6477,13 @@ async fn spawn_task_turn_span_inherits_dispatch_trace_context() {
_ctx: Arc<TurnContext>,
_input: Vec<TurnInput>,
_cancellation_token: CancellationToken,
) -> Option<String> {
) -> SessionTaskResult {
let mut trace = self
.captured_trace
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*trace = current_span_w3c_trace_context();
None
Ok(None)
}
}
@@ -8676,8 +8679,8 @@ impl SessionTask for CompletingTask {
_ctx: Arc<TurnContext>,
_input: Vec<TurnInput>,
_cancellation_token: CancellationToken,
) -> Option<String> {
None
) -> SessionTaskResult {
Ok(None)
}
}
@@ -8702,10 +8705,10 @@ impl SessionTask for NeverEndingTask {
_ctx: Arc<TurnContext>,
_input: Vec<TurnInput>,
cancellation_token: CancellationToken,
) -> Option<String> {
) -> SessionTaskResult {
if self.listen_to_cancellation_token {
cancellation_token.cancelled().await;
return None;
return Ok(None);
}
loop {
sleep(Duration::from_secs(60)).await;
@@ -8731,14 +8734,14 @@ impl SessionTask for GuardianDeniedApprovalTask {
ctx: Arc<TurnContext>,
_input: Vec<TurnInput>,
cancellation_token: CancellationToken,
) -> Option<String> {
) -> SessionTaskResult {
let session = session.clone_session();
for _ in 0..3 {
crate::guardian::record_guardian_denial_for_test(&session, &ctx, &ctx.sub_id).await;
}
cancellation_token.cancelled().await;
None
Ok(None)
}
}
@@ -8961,7 +8964,7 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input()
.await
.expect("steer pending input into active turn");
sess.on_task_finished(Arc::clone(&tc), /*last_agent_message*/ None)
sess.on_task_finished(Arc::clone(&tc), /*task_result*/ Ok(None))
.await;
let history = sess.clone_history().await;
+25 -13
View File
@@ -144,7 +144,7 @@ pub(crate) async fn run_turn(
input: Vec<TurnInput>,
prewarmed_client_session: Option<ModelClientSession>,
cancellation_token: CancellationToken,
) -> Option<String> {
) -> CodexResult<Option<String>> {
let mut client_session =
prewarmed_client_session.unwrap_or_else(|| sess.services.model_client.new_session());
// TODO(ccunningham): Pre-turn compaction runs before context updates and the
@@ -152,25 +152,31 @@ pub(crate) async fn run_turn(
// diffs/full reinjection + user input) and trigger compaction preemptively
// when they would push the thread over the compaction threshold.
if let Err(err) = run_pre_sampling_compact(&sess, &turn_context, &mut client_session).await {
if matches!(err, CodexErr::TurnAborted) {
return Err(err);
}
let error = err.to_codex_protocol_error();
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
.await;
error!("Failed to run pre-sampling compact");
return None;
return Ok(None);
}
sess.record_context_updates_and_set_reference_context_item(turn_context.as_ref())
.await;
let (injection_items, explicitly_enabled_connectors) =
build_skills_and_plugins(&sess, turn_context.as_ref(), &input, &cancellation_token).await?;
let Some((injection_items, explicitly_enabled_connectors)) =
build_skills_and_plugins(&sess, turn_context.as_ref(), &input, &cancellation_token).await
else {
return Ok(None);
};
if run_pending_session_start_hooks(&sess, &turn_context).await {
return None;
return Ok(None);
}
let mut can_drain_pending_input = input.is_empty();
if run_hooks_and_record_inputs(&sess, &turn_context, &input).await {
return None;
return Ok(None);
}
sess.merge_connector_selection(explicitly_enabled_connectors.clone())
@@ -336,10 +342,13 @@ pub(crate) async fn run_turn(
)
.await
{
if matches!(err, CodexErr::TurnAborted) {
return Err(err);
}
let error = err.to_codex_protocol_error();
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
.await;
return None;
return Ok(None);
}
can_drain_pending_input = !model_needs_follow_up;
continue;
@@ -386,15 +395,14 @@ pub(crate) async fn run_turn(
)
.await
{
return None;
return Ok(None);
}
break;
}
continue;
}
Err(CodexErr::TurnAborted) => {
// Aborted turn is reported via a different event.
break;
Err(err @ CodexErr::TurnAborted) => {
return Err(err);
}
Err(codex_error @ CodexErr::InvalidImageRequest()) => {
{
@@ -433,7 +441,7 @@ pub(crate) async fn run_turn(
}
}
last_agent_message
Ok(last_agent_message)
}
#[instrument(level = "trace", skip_all)]
@@ -2197,10 +2205,14 @@ async fn try_run_sampling_request(
&mut assistant_message_stream_parsers,
)
.await;
sess.record_token_usage_info(&turn_context, token_usage.as_ref())
let budget_result = sess
.record_token_usage_info(&turn_context, token_usage.as_ref())
.await;
should_emit_token_count = true;
should_emit_turn_diff = true;
if let Err(err) = budget_result {
break Err(err);
}
if let Some(false) = end_turn {
needs_follow_up = true;
}