mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
[codex] abort turns when rollout budgets expire (token budget 3/3) (#28707)
## Stack Depends on #28494. ## Description This PR propagates shared rollout-budget exhaustion through the existing `CodexErr::TurnAborted` task result. Each thread records its model usage against the same ledger. Once the ledger is exhausted, that usage update and all later usage updates return `TurnAborted`. The task wrapper emits the normal aborted-turn event and lifecycle instead of completing the turn. This is intentionally a soft boundary: there is no cross-thread `Op::Interrupt` fanout. An in-flight thread can finish its current response before it observes the exhausted ledger, but every thread aborts at its next usage-accounting boundary. ## Tests The integration coverage verifies that: - the response that exhausts the budget aborts its turn; - a later response also aborts because the shared ledger remains exhausted; and - sub-agent usage draws from the same shared ledger; and - local and remote-v2 compaction abort without retrying or emitting a generic error. Local checks: - `just test -p codex-core exhausted_budget_aborts_current_and_later_turns` - `just test -p codex-core subagent_usage_draws_from_the_shared_budget` - `just test -p codex-core abort_regular_task_emits_marker_before_turn_aborted` - `just test -p codex-core compaction_budget_exhaustion_aborts_without_error_or_retry` - `just fix -p codex-core` - `just fmt` - `git diff --check` The full workspace test suite was not run locally.
This commit is contained in:
committed by
GitHub
Unverified
parent
7abfcf220b
commit
dac588f413
@@ -3410,17 +3410,19 @@ impl Session {
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
token_usage: Option<&TokenUsage>,
|
||||
) {
|
||||
self.record_token_usage_info(turn_context, token_usage)
|
||||
) -> CodexResult<()> {
|
||||
let result = self
|
||||
.record_token_usage_info(turn_context, token_usage)
|
||||
.await;
|
||||
self.send_token_count_event(turn_context).await;
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) async fn record_token_usage_info(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
token_usage: Option<&TokenUsage>,
|
||||
) {
|
||||
) -> CodexResult<()> {
|
||||
if let Some(token_usage) = token_usage {
|
||||
let token_info = {
|
||||
let mut state = self.state.lock().await;
|
||||
@@ -3434,7 +3436,7 @@ impl Session {
|
||||
}
|
||||
state.token_info()
|
||||
};
|
||||
self.record_rollout_budget_usage(token_usage);
|
||||
let budget_result = self.record_rollout_budget_usage(token_usage);
|
||||
if let Some(token_info) = token_info.as_ref() {
|
||||
for contributor in self.services.extensions.token_usage_contributors() {
|
||||
contributor
|
||||
@@ -3447,7 +3449,9 @@ impl Session {
|
||||
.await;
|
||||
}
|
||||
}
|
||||
budget_result?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn recompute_token_usage(&self, turn_context: &TurnContext) {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
use super::session::Session;
|
||||
use super::turn_context::TurnContext;
|
||||
use crate::context::ContextualUserFragment;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
|
||||
pub(super) async fn maybe_record_reminder(
|
||||
@@ -21,10 +23,15 @@ pub(super) async fn maybe_record_reminder(
|
||||
}
|
||||
|
||||
impl Session {
|
||||
pub(crate) fn record_rollout_budget_usage(&self, usage: &TokenUsage) {
|
||||
self.services
|
||||
pub(crate) fn record_rollout_budget_usage(&self, usage: &TokenUsage) -> CodexResult<()> {
|
||||
if self
|
||||
.services
|
||||
.agent_control
|
||||
.rollout_budget()
|
||||
.record_usage(usage);
|
||||
.record_usage(usage)
|
||||
{
|
||||
return Err(CodexErr::TurnAborted);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,6 +70,7 @@ use crate::state::ActiveTurn;
|
||||
use crate::state::TaskKind;
|
||||
use crate::tasks::SessionTask;
|
||||
use crate::tasks::SessionTaskContext;
|
||||
use crate::tasks::SessionTaskResult;
|
||||
use crate::tasks::UserShellCommandMode;
|
||||
use crate::tasks::execute_user_shell_command;
|
||||
use crate::tools::ToolRouter;
|
||||
@@ -2150,10 +2151,12 @@ async fn record_token_usage_info_notifies_extension_contributors() {
|
||||
|
||||
session
|
||||
.record_token_usage_info(&turn_context, Some(&first_usage))
|
||||
.await;
|
||||
.await
|
||||
.expect("first usage should be recorded");
|
||||
session
|
||||
.record_token_usage_info(&turn_context, Some(&second_usage))
|
||||
.await;
|
||||
.await
|
||||
.expect("second usage should be recorded");
|
||||
|
||||
let mut expected_total_usage = first_usage.clone();
|
||||
expected_total_usage.add_assign(&second_usage);
|
||||
@@ -6474,13 +6477,13 @@ async fn spawn_task_turn_span_inherits_dispatch_trace_context() {
|
||||
_ctx: Arc<TurnContext>,
|
||||
_input: Vec<TurnInput>,
|
||||
_cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> SessionTaskResult {
|
||||
let mut trace = self
|
||||
.captured_trace
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
*trace = current_span_w3c_trace_context();
|
||||
None
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8676,8 +8679,8 @@ impl SessionTask for CompletingTask {
|
||||
_ctx: Arc<TurnContext>,
|
||||
_input: Vec<TurnInput>,
|
||||
_cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
None
|
||||
) -> SessionTaskResult {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8702,10 +8705,10 @@ impl SessionTask for NeverEndingTask {
|
||||
_ctx: Arc<TurnContext>,
|
||||
_input: Vec<TurnInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> SessionTaskResult {
|
||||
if self.listen_to_cancellation_token {
|
||||
cancellation_token.cancelled().await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
loop {
|
||||
sleep(Duration::from_secs(60)).await;
|
||||
@@ -8731,14 +8734,14 @@ impl SessionTask for GuardianDeniedApprovalTask {
|
||||
ctx: Arc<TurnContext>,
|
||||
_input: Vec<TurnInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> SessionTaskResult {
|
||||
let session = session.clone_session();
|
||||
for _ in 0..3 {
|
||||
crate::guardian::record_guardian_denial_for_test(&session, &ctx, &ctx.sub_id).await;
|
||||
}
|
||||
|
||||
cancellation_token.cancelled().await;
|
||||
None
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8961,7 +8964,7 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input()
|
||||
.await
|
||||
.expect("steer pending input into active turn");
|
||||
|
||||
sess.on_task_finished(Arc::clone(&tc), /*last_agent_message*/ None)
|
||||
sess.on_task_finished(Arc::clone(&tc), /*task_result*/ Ok(None))
|
||||
.await;
|
||||
|
||||
let history = sess.clone_history().await;
|
||||
|
||||
@@ -144,7 +144,7 @@ pub(crate) async fn run_turn(
|
||||
input: Vec<TurnInput>,
|
||||
prewarmed_client_session: Option<ModelClientSession>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> CodexResult<Option<String>> {
|
||||
let mut client_session =
|
||||
prewarmed_client_session.unwrap_or_else(|| sess.services.model_client.new_session());
|
||||
// TODO(ccunningham): Pre-turn compaction runs before context updates and the
|
||||
@@ -152,25 +152,31 @@ pub(crate) async fn run_turn(
|
||||
// diffs/full reinjection + user input) and trigger compaction preemptively
|
||||
// when they would push the thread over the compaction threshold.
|
||||
if let Err(err) = run_pre_sampling_compact(&sess, &turn_context, &mut client_session).await {
|
||||
if matches!(err, CodexErr::TurnAborted) {
|
||||
return Err(err);
|
||||
}
|
||||
let error = err.to_codex_protocol_error();
|
||||
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
|
||||
.await;
|
||||
error!("Failed to run pre-sampling compact");
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
sess.record_context_updates_and_set_reference_context_item(turn_context.as_ref())
|
||||
.await;
|
||||
|
||||
let (injection_items, explicitly_enabled_connectors) =
|
||||
build_skills_and_plugins(&sess, turn_context.as_ref(), &input, &cancellation_token).await?;
|
||||
let Some((injection_items, explicitly_enabled_connectors)) =
|
||||
build_skills_and_plugins(&sess, turn_context.as_ref(), &input, &cancellation_token).await
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if run_pending_session_start_hooks(&sess, &turn_context).await {
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
let mut can_drain_pending_input = input.is_empty();
|
||||
if run_hooks_and_record_inputs(&sess, &turn_context, &input).await {
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
sess.merge_connector_selection(explicitly_enabled_connectors.clone())
|
||||
@@ -336,10 +342,13 @@ pub(crate) async fn run_turn(
|
||||
)
|
||||
.await
|
||||
{
|
||||
if matches!(err, CodexErr::TurnAborted) {
|
||||
return Err(err);
|
||||
}
|
||||
let error = err.to_codex_protocol_error();
|
||||
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
can_drain_pending_input = !model_needs_follow_up;
|
||||
continue;
|
||||
@@ -386,15 +395,14 @@ pub(crate) async fn run_turn(
|
||||
)
|
||||
.await
|
||||
{
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(CodexErr::TurnAborted) => {
|
||||
// Aborted turn is reported via a different event.
|
||||
break;
|
||||
Err(err @ CodexErr::TurnAborted) => {
|
||||
return Err(err);
|
||||
}
|
||||
Err(codex_error @ CodexErr::InvalidImageRequest()) => {
|
||||
{
|
||||
@@ -433,7 +441,7 @@ pub(crate) async fn run_turn(
|
||||
}
|
||||
}
|
||||
|
||||
last_agent_message
|
||||
Ok(last_agent_message)
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip_all)]
|
||||
@@ -2197,10 +2205,14 @@ async fn try_run_sampling_request(
|
||||
&mut assistant_message_stream_parsers,
|
||||
)
|
||||
.await;
|
||||
sess.record_token_usage_info(&turn_context, token_usage.as_ref())
|
||||
let budget_result = sess
|
||||
.record_token_usage_info(&turn_context, token_usage.as_ref())
|
||||
.await;
|
||||
should_emit_token_count = true;
|
||||
should_emit_turn_diff = true;
|
||||
if let Err(err) = budget_result {
|
||||
break Err(err);
|
||||
}
|
||||
if let Some(false) = end_turn {
|
||||
needs_follow_up = true;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user