agent/codex
2
0
Fork 0
mirror of https://github.com/pchuan98/codex.git synced 2026-07-01 00:31:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Make thread store process-scoped (#19474)

Browse Source 
- Build one app-server process ThreadStore from startup config and share
it with ThreadManager and CodexMessageProcessor.
- Remove per-thread/fork store reconstruction so effective thread config
cannot switch the persistence backend.
- Add params to ThreadStore create/resume for specifying thread
metadata, since otherwise the metadata from store creation would be used
(incorrectly).
This commit is contained in:
Tom
2026-04-30 21:24:59 -07:00
committed by GitHub
parent f50c02d7bc
commit fe05acad23
55 changed files with 1076 additions and 514 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/rollout/src/metadata.rs
+13 -13
View File
@@ -1,6 +1,5 @@
use crate::ARCHIVED_SESSIONS_SUBDIR;
use crate::SESSIONS_SUBDIR;
use crate::config::RolloutConfigView;
use crate::list;
use crate::list::parse_timestamp_uuid_from_filename;
use crate::recorder::RolloutRecorder;
@@ -135,7 +134,8 @@ pub async fn extract_metadata_from_rollout(
pub(crate) async fn backfill_sessions(
runtime: &codex_state::StateRuntime,
config: &impl RolloutConfigView,
codex_home: &Path,
default_provider: &str,
) {
let metric_client = codex_otel::global();
let timer = metric_client
@@ -146,7 +146,7 @@ pub(crate) async fn backfill_sessions(
Err(err) => {
warn!(
"failed to read backfill state at {}: {err}",
config.codex_home().display()
codex_home.display()
);
BackfillState::default()
}
@@ -159,7 +159,7 @@ pub(crate) async fn backfill_sessions(
Err(err) => {
warn!(
"failed to claim backfill worker at {}: {err}",
config.codex_home().display()
codex_home.display()
);
return;
}
@@ -167,7 +167,7 @@ pub(crate) async fn backfill_sessions(
if !claimed {
info!(
"state db backfill already running at {}; skipping duplicate worker",
config.codex_home().display()
codex_home.display()
);
return;
}
@@ -176,7 +176,7 @@ pub(crate) async fn backfill_sessions(
Err(err) => {
warn!(
"failed to read claimed backfill state at {}: {err}",
config.codex_home().display()
codex_home.display()
);
BackfillState {
status: BackfillStatus::Running,
@@ -188,15 +188,15 @@ pub(crate) async fn backfill_sessions(
if let Err(err) = runtime.mark_backfill_running().await {
warn!(
"failed to mark backfill running at {}: {err}",
config.codex_home().display()
codex_home.display()
);
} else {
backfill_state.status = BackfillStatus::Running;
}
}
let sessions_root = config.codex_home().join(SESSIONS_SUBDIR);
let archived_root = config.codex_home().join(ARCHIVED_SESSIONS_SUBDIR);
let sessions_root = codex_home.join(SESSIONS_SUBDIR);
let archived_root = codex_home.join(ARCHIVED_SESSIONS_SUBDIR);
let mut rollout_paths: Vec<BackfillRolloutPath> = Vec::new();
for (root, archived) in [(sessions_root, false), (archived_root, true)] {
if !tokio::fs::try_exists(&root).await.unwrap_or(false) {
@@ -205,7 +205,7 @@ pub(crate) async fn backfill_sessions(
match collect_rollout_paths(&root).await {
Ok(paths) => {
rollout_paths.extend(paths.into_iter().map(|path| BackfillRolloutPath {
watermark: backfill_watermark_for_path(config.codex_home(), &path),
watermark: backfill_watermark_for_path(codex_home, &path),
path,
archived,
}));
@@ -232,7 +232,7 @@ pub(crate) async fn backfill_sessions(
for batch in rollout_paths.chunks(BACKFILL_BATCH_SIZE) {
for rollout in batch {
stats.scanned = stats.scanned.saturating_add(1);
match extract_metadata_from_rollout(&rollout.path, config.model_provider_id()).await {
match extract_metadata_from_rollout(&rollout.path, default_provider).await {
Ok(outcome) => {
if outcome.parse_errors > 0
&& let Some(ref metric_client) = metric_client
@@ -309,7 +309,7 @@ pub(crate) async fn backfill_sessions(
{
warn!(
"failed to checkpoint backfill at {}: {err}",
config.codex_home().display()
codex_home.display()
);
} else {
last_watermark = Some(last_entry.watermark.clone());
@@ -322,7 +322,7 @@ pub(crate) async fn backfill_sessions(
{
warn!(
"failed to mark backfill complete at {}: {err}",
config.codex_home().display()
codex_home.display()
);
}
codex-rs/rollout/src/metadata_tests.rs
+3 -17
View File
@@ -1,7 +1,6 @@
#![allow(warnings, clippy::all)]
use super::*;
use crate::config::RolloutConfig;
use chrono::DateTime;
use chrono::NaiveDateTime;
use chrono::Timelike;
@@ -24,16 +23,6 @@ use std::path::PathBuf;
use tempfile::tempdir;
use uuid::Uuid;
fn test_config(codex_home: PathBuf) -> RolloutConfig {
RolloutConfig {
sqlite_home: codex_home.clone(),
cwd: codex_home.clone(),
codex_home,
model_provider_id: "test-provider".to_string(),
generate_memories: true,
}
}
#[tokio::test]
async fn extract_metadata_from_rollout_uses_session_meta() {
let dir = tempdir().expect("tempdir");
@@ -210,8 +199,7 @@ async fn backfill_sessions_resumes_from_watermark_and_marks_complete() {
))
.await;
let config = test_config(codex_home.clone());
backfill_sessions(runtime.as_ref(), &config).await;
backfill_sessions(runtime.as_ref(), codex_home.as_path(), "test-provider").await;
let first_id = ThreadId::from_string(&first_uuid.to_string()).expect("first thread id");
let second_id = ThreadId::from_string(&second_uuid.to_string()).expect("second thread id");
@@ -278,8 +266,7 @@ async fn backfill_sessions_preserves_existing_git_branch_and_fills_missing_git_f
.await
.expect("existing metadata upsert");
let config = test_config(codex_home.clone());
backfill_sessions(runtime.as_ref(), &config).await;
backfill_sessions(runtime.as_ref(), codex_home.as_path(), "test-provider").await;
let persisted = runtime
.get_thread(thread_id)
@@ -313,8 +300,7 @@ async fn backfill_sessions_normalizes_cwd_before_upsert() {
.await
.expect("initialize runtime");
let config = test_config(codex_home.clone());
backfill_sessions(runtime.as_ref(), &config).await;
backfill_sessions(runtime.as_ref(), codex_home.as_path(), "test-provider").await;
let thread_id = ThreadId::from_string(&thread_uuid.to_string()).expect("thread id");
let stored = runtime
codex-rs/rollout/src/state_db.rs
+28 -7
View File
@@ -25,9 +25,23 @@ pub type StateDbHandle = Arc<codex_state::StateRuntime>;
/// Initialize the state runtime for thread state persistence and backfill checks.
pub async fn init(config: &impl RolloutConfigView) -> Option<StateDbHandle> {
let config = RolloutConfig::from_view(config);
init_with_roots(
config.codex_home,
config.sqlite_home,
config.model_provider_id,
)
.await
}
/// Initialize the state runtime for a local thread store.
pub async fn init_with_roots(
codex_home: PathBuf,
sqlite_home: PathBuf,
default_model_provider_id: String,
) -> Option<StateDbHandle> {
let runtime = match codex_state::StateRuntime::init(
config.sqlite_home.clone(),
config.model_provider_id.clone(),
sqlite_home.clone(),
default_model_provider_id.clone(),
)
.await
{
@@ -35,7 +49,7 @@ pub async fn init(config: &impl RolloutConfigView) -> Option<StateDbHandle> {
Err(err) => {
warn!(
"failed to initialize state runtime at {}: {err}",
config.sqlite_home.display()
sqlite_home.display()
);
return None;
}
@@ -45,16 +59,20 @@ pub async fn init(config: &impl RolloutConfigView) -> Option<StateDbHandle> {
Err(err) => {
warn!(
"failed to read backfill state at {}: {err}",
config.codex_home.display()
codex_home.display()
);
return None;
}
};
if backfill_state.status != codex_state::BackfillStatus::Complete {
let runtime_for_backfill = runtime.clone();
let config = config.clone();
tokio::spawn(async move {
metadata::backfill_sessions(runtime_for_backfill.as_ref(), &config).await;
metadata::backfill_sessions(
runtime_for_backfill.as_ref(),
codex_home.as_path(),
default_model_provider_id.as_str(),
)
.await;
});
}
Some(runtime)
@@ -487,7 +505,7 @@ pub async fn read_repair_rollout_path(
pub async fn apply_rollout_items(
context: Option<&codex_state::StateRuntime>,
rollout_path: &Path,
_default_provider: &str,
default_provider: &str,
builder: Option<&ThreadMetadataBuilder>,
items: &[RolloutItem],
stage: &str,
@@ -511,6 +529,9 @@ pub async fn apply_rollout_items(
}
},
};
if builder.model_provider.is_none() {
builder.model_provider = Some(default_provider.to_string());
}
builder.rollout_path = rollout_path.to_path_buf();
builder.cwd = normalize_cwd_for_state_db(&builder.cwd);
if let Err(err) = ctx