feat: prevent double backfill (#11377)

## Summary

Add a DB-backed lease to prevent duplicate `.sqlite` backfill workers
from running concurrently.

### What changed
- Added StateRuntime::try_claim_backfill(lease_seconds) that atomically
claims backfill only when:
  - backfill is not complete, and
  - no fresh running worker currently owns it.
- Updated backfill_sessions to use the claim API and exit early when
another worker already holds the lease.
- Added runtime tests covering:
  - singleton claim behavior,
  - stale lease takeover,
  - claim blocked after complete.
- Set backfill lease to 900s in production and 1s in tests.

### Why

This avoids duplicate backfill work and reduces backfill status churn
under concurrent startup, while preserving
current best-effort fallback behavior.
This commit is contained in:
jif-oai
2026-02-11 00:24:20 +00:00
committed by GitHub
parent 674799d356
commit 87bbfc50a1
2 changed files with 145 additions and 11 deletions
+80
View File
@@ -108,6 +108,34 @@ WHERE id = 1
crate::BackfillState::try_from_row(&row)
}
/// Attempt to claim ownership of rollout metadata backfill.
///
/// Returns `true` when this runtime claimed the backfill worker slot.
/// Returns `false` if backfill is already complete or currently owned by a
/// non-expired worker.
pub async fn try_claim_backfill(&self, lease_seconds: i64) -> anyhow::Result<bool> {
self.ensure_backfill_state_row().await?;
let now = Utc::now().timestamp();
let lease_cutoff = now.saturating_sub(lease_seconds.max(0));
let result = sqlx::query(
r#"
UPDATE backfill_state
SET status = ?, updated_at = ?
WHERE id = 1
AND status != ?
AND (status != ? OR updated_at <= ?)
"#,
)
.bind(crate::BackfillStatus::Running.as_str())
.bind(now)
.bind(crate::BackfillStatus::Complete.as_str())
.bind(crate::BackfillStatus::Running.as_str())
.bind(lease_cutoff)
.execute(self.pool.as_ref())
.await?;
Ok(result.rows_affected() == 1)
}
/// Mark rollout metadata backfill as running.
pub async fn mark_backfill_running(&self) -> anyhow::Result<()> {
self.ensure_backfill_state_row().await?;
@@ -1028,6 +1056,58 @@ mod tests {
let _ = tokio::fs::remove_dir_all(codex_home).await;
}
#[tokio::test]
async fn backfill_claim_is_singleton_until_stale_and_blocked_when_complete() {
let codex_home = unique_temp_dir();
let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None)
.await
.expect("initialize runtime");
let claimed = runtime
.try_claim_backfill(3600)
.await
.expect("initial backfill claim");
assert_eq!(claimed, true);
let duplicate_claim = runtime
.try_claim_backfill(3600)
.await
.expect("duplicate backfill claim");
assert_eq!(duplicate_claim, false);
let stale_updated_at = Utc::now().timestamp().saturating_sub(10_000);
sqlx::query(
r#"
UPDATE backfill_state
SET status = ?, updated_at = ?
WHERE id = 1
"#,
)
.bind(crate::BackfillStatus::Running.as_str())
.bind(stale_updated_at)
.execute(runtime.pool.as_ref())
.await
.expect("force stale backfill lease");
let stale_claim = runtime
.try_claim_backfill(10)
.await
.expect("stale backfill claim");
assert_eq!(stale_claim, true);
runtime
.mark_backfill_complete(None)
.await
.expect("mark complete");
let claim_after_complete = runtime
.try_claim_backfill(3600)
.await
.expect("claim after complete");
assert_eq!(claim_after_complete, false);
let _ = tokio::fs::remove_dir_all(codex_home).await;
}
#[tokio::test]
async fn stage1_claim_skips_when_up_to_date() {
let codex_home = unique_temp_dir();