mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
[codex] abort turns when rollout budgets expire (token budget 3/3) (#28707)
## Stack Depends on #28494. ## Description This PR propagates shared rollout-budget exhaustion through the existing `CodexErr::TurnAborted` task result. Each thread records its model usage against the same ledger. Once the ledger is exhausted, that usage update and all later usage updates return `TurnAborted`. The task wrapper emits the normal aborted-turn event and lifecycle instead of completing the turn. This is intentionally a soft boundary: there is no cross-thread `Op::Interrupt` fanout. An in-flight thread can finish its current response before it observes the exhausted ledger, but every thread aborts at its next usage-accounting boundary. ## Tests The integration coverage verifies that: - the response that exhausts the budget aborts its turn; - a later response also aborts because the shared ledger remains exhausted; and - sub-agent usage draws from the same shared ledger; and - local and remote-v2 compaction abort without retrying or emitting a generic error. Local checks: - `just test -p codex-core exhausted_budget_aborts_current_and_later_turns` - `just test -p codex-core subagent_usage_draws_from_the_shared_budget` - `just test -p codex-core abort_regular_task_emits_marker_before_turn_aborted` - `just test -p codex-core compaction_budget_exhaustion_aborts_without_error_or_retry` - `just fix -p codex-core` - `just fmt` - `git diff --check` The full workspace test suite was not run locally.
This commit is contained in:
committed by
GitHub
Unverified
parent
7abfcf220b
commit
dac588f413
@@ -2,10 +2,12 @@ use std::sync::Arc;
|
||||
|
||||
use super::SessionTask;
|
||||
use super::SessionTaskContext;
|
||||
use super::SessionTaskResult;
|
||||
use super::emit_compact_metric;
|
||||
use crate::session::TurnInput;
|
||||
use crate::session::turn_context::TurnContext;
|
||||
use crate::state::TaskKind;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -27,9 +29,9 @@ impl SessionTask for CompactTask {
|
||||
ctx: Arc<TurnContext>,
|
||||
_input: Vec<TurnInput>,
|
||||
_cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> SessionTaskResult {
|
||||
let session = session.clone_session();
|
||||
let _ = if crate::compact::should_use_remote_compact_task(ctx.provider.info()) {
|
||||
let result = if crate::compact::should_use_remote_compact_task(ctx.provider.info()) {
|
||||
if ctx
|
||||
.config
|
||||
.features
|
||||
@@ -67,6 +69,9 @@ impl SessionTask for CompactTask {
|
||||
}];
|
||||
crate::compact::run_compact_task(session.clone(), ctx, input).await
|
||||
};
|
||||
None
|
||||
if let Err(err @ CodexErr::TurnAborted) = result {
|
||||
return Err(err);
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,6 +54,8 @@ use codex_protocol::protocol::TurnCompleteEvent;
|
||||
use codex_protocol::protocol::WarningEvent;
|
||||
|
||||
use codex_features::Feature;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::models::ContentItem;
|
||||
pub(crate) use compact::CompactTask;
|
||||
pub(crate) use regular::RegularTask;
|
||||
@@ -65,6 +67,8 @@ pub(crate) use user_shell::execute_user_shell_command;
|
||||
const GRACEFULL_INTERRUPTION_TIMEOUT_MS: u64 = 100;
|
||||
const TASK_COMPACT_METRIC: &str = "codex.task.compact";
|
||||
|
||||
pub(crate) type SessionTaskResult = CodexResult<Option<String>>;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum InterruptedTurnHistoryMarker {
|
||||
Disabled,
|
||||
@@ -222,14 +226,16 @@ pub(crate) trait SessionTask: Send + Sync + 'static {
|
||||
/// provided `cancellation_token` is cancelled when the session requests an
|
||||
/// abort; implementers should watch for it and terminate quickly once it
|
||||
/// fires. Returning [`Some`] yields a final message that
|
||||
/// [`Session::on_task_finished`] will emit to the client.
|
||||
/// [`Session::on_task_finished`] will emit to the client. Returning
|
||||
/// [`CodexErr::TurnAborted`] completes the task through the aborted-turn
|
||||
/// lifecycle instead.
|
||||
fn run(
|
||||
self: Arc<Self>,
|
||||
session: Arc<SessionTaskContext>,
|
||||
ctx: Arc<TurnContext>,
|
||||
input: Vec<TurnInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> impl std::future::Future<Output = Option<String>> + Send;
|
||||
) -> impl std::future::Future<Output = SessionTaskResult> + Send;
|
||||
|
||||
/// Gives the task a chance to perform cleanup after an abort.
|
||||
///
|
||||
@@ -258,7 +264,7 @@ pub(crate) trait AnySessionTask: Send + Sync + 'static {
|
||||
ctx: Arc<TurnContext>,
|
||||
input: Vec<TurnInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> BoxFuture<'static, Option<String>>;
|
||||
) -> BoxFuture<'static, SessionTaskResult>;
|
||||
|
||||
fn abort<'a>(
|
||||
&'a self,
|
||||
@@ -285,7 +291,7 @@ where
|
||||
ctx: Arc<TurnContext>,
|
||||
input: Vec<TurnInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> BoxFuture<'static, Option<String>> {
|
||||
) -> BoxFuture<'static, SessionTaskResult> {
|
||||
Box::pin(SessionTask::run(
|
||||
self,
|
||||
session,
|
||||
@@ -395,7 +401,7 @@ impl Session {
|
||||
let handle = tokio::spawn(
|
||||
async move {
|
||||
let ctx_for_finish = Arc::clone(&ctx);
|
||||
let last_agent_message = task_for_run
|
||||
let task_result = task_for_run
|
||||
.run(
|
||||
Arc::clone(&session_ctx),
|
||||
ctx,
|
||||
@@ -418,8 +424,8 @@ impl Session {
|
||||
.await;
|
||||
}
|
||||
if !task_cancellation_token.is_cancelled() {
|
||||
// Emit completion uniformly from spawn site so all tasks share the same lifecycle.
|
||||
sess.on_task_finished(Arc::clone(&ctx_for_finish), last_agent_message)
|
||||
// Finish uniformly from the spawn site so all tasks share the same lifecycle.
|
||||
sess.on_task_finished(Arc::clone(&ctx_for_finish), task_result)
|
||||
.await;
|
||||
}
|
||||
done_clone.notify_waiters();
|
||||
@@ -557,8 +563,16 @@ impl Session {
|
||||
pub async fn on_task_finished(
|
||||
self: &Arc<Self>,
|
||||
turn_context: Arc<TurnContext>,
|
||||
last_agent_message: Option<String>,
|
||||
task_result: SessionTaskResult,
|
||||
) {
|
||||
let (last_agent_message, abort_reason) = match task_result {
|
||||
Ok(last_agent_message) => (last_agent_message, None),
|
||||
Err(CodexErr::TurnAborted) => (None, Some(TurnAbortReason::Interrupted)),
|
||||
Err(err) => {
|
||||
warn!(%err, "session task returned an unexpected error");
|
||||
(None, None)
|
||||
}
|
||||
};
|
||||
turn_context
|
||||
.turn_metadata_state
|
||||
.cancel_git_enrichment_task();
|
||||
@@ -730,25 +744,36 @@ impl Session {
|
||||
.turn_timing_state
|
||||
.completed_at_and_duration_ms()
|
||||
.await;
|
||||
let time_to_first_token_ms = turn_context
|
||||
.turn_timing_state
|
||||
.time_to_first_token_ms()
|
||||
.await;
|
||||
self.services
|
||||
.analytics_events_client
|
||||
.track_turn_profile(TurnProfileFact {
|
||||
turn_id: turn_context.sub_id.clone(),
|
||||
profile: turn_context.turn_timing_state.complete_profile(),
|
||||
});
|
||||
self.emit_turn_stop_lifecycle(turn_context.extension_data.as_ref())
|
||||
.await;
|
||||
let event = EventMsg::TurnComplete(TurnCompleteEvent {
|
||||
turn_id: turn_context.sub_id.clone(),
|
||||
last_agent_message,
|
||||
completed_at,
|
||||
duration_ms,
|
||||
time_to_first_token_ms,
|
||||
});
|
||||
let event = if let Some(reason) = abort_reason {
|
||||
self.emit_turn_abort_lifecycle(reason.clone(), turn_context.extension_data.as_ref())
|
||||
.await;
|
||||
EventMsg::TurnAborted(TurnAbortedEvent {
|
||||
turn_id: Some(turn_context.sub_id.clone()),
|
||||
reason,
|
||||
completed_at,
|
||||
duration_ms,
|
||||
})
|
||||
} else {
|
||||
let time_to_first_token_ms = turn_context
|
||||
.turn_timing_state
|
||||
.time_to_first_token_ms()
|
||||
.await;
|
||||
self.emit_turn_stop_lifecycle(turn_context.extension_data.as_ref())
|
||||
.await;
|
||||
EventMsg::TurnComplete(TurnCompleteEvent {
|
||||
turn_id: turn_context.sub_id.clone(),
|
||||
last_agent_message,
|
||||
completed_at,
|
||||
duration_ms,
|
||||
time_to_first_token_ms,
|
||||
})
|
||||
};
|
||||
self.send_event(turn_context.as_ref(), event).await;
|
||||
self.services
|
||||
.guardian_rejection_circuit_breaker
|
||||
|
||||
@@ -14,6 +14,7 @@ use tracing::trace_span;
|
||||
|
||||
use super::SessionTask;
|
||||
use super::SessionTaskContext;
|
||||
use super::SessionTaskResult;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RegularTask;
|
||||
@@ -39,7 +40,7 @@ impl SessionTask for RegularTask {
|
||||
ctx: Arc<TurnContext>,
|
||||
input: Vec<TurnInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> SessionTaskResult {
|
||||
let sess = session.clone_session();
|
||||
let turn_extension_data = session.turn_extension_data();
|
||||
let run_turn_span = trace_span!("run_turn");
|
||||
@@ -61,7 +62,7 @@ impl SessionTask for RegularTask {
|
||||
.instrument(trace_span!("regular_task.prepare_run_turn"))
|
||||
.await;
|
||||
let prewarmed_client_session = match prewarmed_client_session {
|
||||
SessionStartupPrewarmResolution::Cancelled => return None,
|
||||
SessionStartupPrewarmResolution::Cancelled => return Ok(None),
|
||||
SessionStartupPrewarmResolution::Unavailable { .. } => None,
|
||||
SessionStartupPrewarmResolution::Ready(prewarmed_client_session) => {
|
||||
Some(*prewarmed_client_session)
|
||||
@@ -79,9 +80,9 @@ impl SessionTask for RegularTask {
|
||||
cancellation_token.child_token(),
|
||||
)
|
||||
.instrument(run_turn_span.clone())
|
||||
.await;
|
||||
.await?;
|
||||
if !sess.input_queue.has_pending_input(&sess.active_turn).await {
|
||||
return last_agent_message;
|
||||
return Ok(last_agent_message);
|
||||
}
|
||||
next_input = Vec::new();
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ use codex_protocol::user_input::UserInput;
|
||||
|
||||
use super::SessionTask;
|
||||
use super::SessionTaskContext;
|
||||
use super::SessionTaskResult;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct ReviewTask;
|
||||
@@ -54,7 +55,7 @@ impl SessionTask for ReviewTask {
|
||||
ctx: Arc<TurnContext>,
|
||||
input: Vec<TurnInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> SessionTaskResult {
|
||||
session.session.services.session_telemetry.counter(
|
||||
"codex.task.review",
|
||||
/*inc*/ 1,
|
||||
@@ -84,7 +85,7 @@ impl SessionTask for ReviewTask {
|
||||
if !cancellation_token.is_cancelled() {
|
||||
exit_review_mode(session.clone_session(), output.clone(), ctx.clone()).await;
|
||||
}
|
||||
None
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn abort(&self, session: Arc<SessionTaskContext>, ctx: Arc<TurnContext>) {
|
||||
|
||||
@@ -41,6 +41,7 @@ use codex_shell_command::parse_command::parse_command;
|
||||
|
||||
use super::SessionTask;
|
||||
use super::SessionTaskContext;
|
||||
use super::SessionTaskResult;
|
||||
use crate::session::session::Session;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
|
||||
@@ -82,7 +83,7 @@ impl SessionTask for UserShellCommandTask {
|
||||
turn_context: Arc<TurnContext>,
|
||||
_input: Vec<TurnInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> SessionTaskResult {
|
||||
execute_user_shell_command(
|
||||
session.clone_session(),
|
||||
turn_context,
|
||||
@@ -91,7 +92,7 @@ impl SessionTask for UserShellCommandTask {
|
||||
UserShellCommandMode::StandaloneTurn,
|
||||
)
|
||||
.await;
|
||||
None
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user