feat: make rollout recorder reliable against errors (#17214)

The rollout writer now keeps an owned/monitored task handle, returns
real Result acks for flush/persist/shutdown, retries failed flushes by
reopening the rollout file, and keeps buffered items until they are
successfully written. Session flushes are now real durability barriers
for fork/rollback/read-after-write paths, while turn completion surfaces
a warning if the rollout still cannot be saved after recovery.
This commit is contained in:
jif-oai
2026-04-10 14:12:33 +01:00
committed by GitHub
parent 085ffb4456
commit 8035cb03f1
10 changed files with 536 additions and 191 deletions
+22 -11
View File
@@ -2216,16 +2216,16 @@ impl Session {
self.services.state_db.clone()
}
/// Ensure rollout file writes are durably flushed.
pub(crate) async fn flush_rollout(&self) {
/// Flush rollout writes and return the final durability-barrier result.
pub(crate) async fn flush_rollout(&self) -> std::io::Result<()> {
let recorder = {
let guard = self.services.rollout.lock().await;
guard.clone()
};
if let Some(rec) = recorder
&& let Err(e) = rec.flush().await
{
warn!("failed to flush rollout recorder: {e}");
if let Some(recorder) = recorder {
recorder.flush().await
} else {
Ok(())
}
}
@@ -2372,7 +2372,7 @@ impl Session {
// Defer seeding the session's initial context until the first turn starts so
// turn/start overrides can be merged before we write to the rollout.
if !is_subagent {
self.flush_rollout().await;
let _ = self.flush_rollout().await;
}
}
InitialHistory::Forked(rollout_items) => {
@@ -2396,7 +2396,7 @@ impl Session {
// Flush after seeding history and any persisted rollout copy.
if !is_subagent {
self.flush_rollout().await;
let _ = self.flush_rollout().await;
}
}
}
@@ -5600,13 +5600,24 @@ mod handlers {
.into_iter()
.chain(std::iter::once(RolloutItem::EventMsg(rollback_msg.clone())))
.collect::<Vec<_>>();
sess.persist_rollout_items(&[RolloutItem::EventMsg(rollback_msg.clone())])
.await;
sess.flush_rollout().await;
sess.apply_rollout_reconstruction(turn_context.as_ref(), replay_items.as_slice())
.await;
sess.recompute_token_usage(turn_context.as_ref()).await;
sess.persist_rollout_items(&[RolloutItem::EventMsg(rollback_msg.clone())])
.await;
if let Err(err) = sess.flush_rollout().await {
sess.send_event(
turn_context.as_ref(),
EventMsg::Warning(WarningEvent {
message: format!(
"Rolled the thread back, but failed to save the rollback marker. Codex will continue retrying. Error: {err}"
),
}),
)
.await;
}
sess.deliver_event_raw(Event {
id: turn_context.sub_id.clone(),
msg: rollback_msg,