Extract rollout into its own crate (#15548)

This commit is contained in:
Ahmed Ibrahim
2026-03-24 18:10:53 -07:00
committed by GitHub
Unverified
parent ea3f3467e2
commit 2e03d8b4d2
32 changed files with 475 additions and 168 deletions
+37 -2
View File
@@ -1881,7 +1881,6 @@ dependencies = [
"codex-exec-server",
"codex-execpolicy",
"codex-features",
"codex-file-search",
"codex-git-utils",
"codex-hooks",
"codex-login",
@@ -1889,6 +1888,7 @@ dependencies = [
"codex-otel",
"codex-protocol",
"codex-rmcp-client",
"codex-rollout",
"codex-sandboxing",
"codex-secrets",
"codex-shell-command",
@@ -1903,6 +1903,7 @@ dependencies = [
"codex-utils-home-dir",
"codex-utils-image",
"codex-utils-output-truncation",
"codex-utils-path",
"codex-utils-pty",
"codex-utils-readiness",
"codex-utils-stream-parser",
@@ -1950,7 +1951,6 @@ dependencies = [
"test-case",
"test-log",
"thiserror 2.0.18",
"time",
"tokio",
"tokio-tungstenite",
"tokio-util",
@@ -2469,6 +2469,31 @@ dependencies = [
"which 8.0.0",
]
[[package]]
name = "codex-rollout"
version = "0.0.0"
dependencies = [
"anyhow",
"async-trait",
"chrono",
"codex-file-search",
"codex-git-utils",
"codex-login",
"codex-otel",
"codex-protocol",
"codex-state",
"codex-utils-path",
"codex-utils-string",
"pretty_assertions",
"serde",
"serde_json",
"tempfile",
"time",
"tokio",
"tracing",
"uuid",
]
[[package]]
name = "codex-sandboxing"
version = "0.0.0"
@@ -2902,6 +2927,16 @@ dependencies = [
"pretty_assertions",
]
[[package]]
name = "codex-utils-path"
version = "0.0.0"
dependencies = [
"codex-utils-absolute-path",
"dunce",
"pretty_assertions",
"tempfile",
]
[[package]]
name = "codex-utils-pty"
version = "0.0.0"
+4
View File
@@ -40,6 +40,7 @@ members = [
"ollama",
"process-hardening",
"protocol",
"rollout",
"rmcp-client",
"responses-api-proxy",
"sandboxing",
@@ -66,6 +67,7 @@ members = [
"utils/approval-presets",
"utils/oss",
"utils/output-truncation",
"utils/path-utils",
"utils/fuzzy-match",
"utils/stream-parser",
"codex-client",
@@ -130,6 +132,7 @@ codex-ollama = { path = "ollama" }
codex-otel = { path = "otel" }
codex-process-hardening = { path = "process-hardening" }
codex-protocol = { path = "protocol" }
codex-rollout = { path = "rollout" }
codex-responses-api-proxy = { path = "responses-api-proxy" }
codex-rmcp-client = { path = "rmcp-client" }
codex-sandboxing = { path = "sandboxing" }
@@ -156,6 +159,7 @@ codex-utils-image = { path = "utils/image" }
codex-utils-json-to-toml = { path = "utils/json-to-toml" }
codex-utils-oss = { path = "utils/oss" }
codex-utils-output-truncation = { path = "utils/output-truncation" }
codex-utils-path = { path = "utils/path-utils" }
codex-utils-pty = { path = "utils/pty" }
codex-utils-readiness = { path = "utils/readiness" }
codex-utils-rustls-provider = { path = "utils/rustls-provider" }
+2 -7
View File
@@ -40,13 +40,13 @@ codex-login = { workspace = true }
codex-shell-command = { workspace = true }
codex-skills = { workspace = true }
codex-execpolicy = { workspace = true }
codex-file-search = { workspace = true }
codex-git-utils = { workspace = true }
codex-hooks = { workspace = true }
codex-network-proxy = { workspace = true }
codex-otel = { workspace = true }
codex-artifacts = { workspace = true }
codex-protocol = { workspace = true }
codex-rollout = { workspace = true }
codex-rmcp-client = { workspace = true }
codex-sandboxing = { workspace = true }
codex-state = { workspace = true }
@@ -56,6 +56,7 @@ codex-utils-cache = { workspace = true }
codex-utils-image = { workspace = true }
codex-utils-home-dir = { workspace = true }
codex-utils-output-truncation = { workspace = true }
codex-utils-path = { workspace = true }
codex-utils-pty = { workspace = true }
codex-utils-readiness = { workspace = true }
codex-secrets = { workspace = true }
@@ -95,12 +96,6 @@ similar = { workspace = true }
tempfile = { workspace = true }
test-log = { workspace = true }
thiserror = { workspace = true }
time = { workspace = true, features = [
"formatting",
"parsing",
"local-offset",
"macros",
] }
tokio = { workspace = true, features = [
"io-std",
"macros",
+1 -1
View File
@@ -1570,7 +1570,7 @@ impl Session {
})?;
let rollout_path = rollout_recorder
.as_ref()
.map(|rec| rec.rollout_path.clone());
.map(|rec| rec.rollout_path().to_path_buf());
let mut post_session_configured_events = Vec::<Event>::new();
+7 -3
View File
@@ -32,7 +32,7 @@ pub mod connectors;
mod context_manager;
mod contextual_user_message;
pub mod custom_prompts;
pub mod env;
pub use codex_utils_path::env;
mod environment_context;
pub mod error;
pub mod exec;
@@ -65,7 +65,8 @@ pub mod mention_syntax;
mod mentions;
pub mod message_history;
mod model_provider_info;
pub mod path_utils;
pub mod utils;
pub use utils::path_utils;
pub mod personality_migration;
pub mod plugins;
mod sandbox_tags;
@@ -123,11 +124,14 @@ pub mod project_doc;
mod rollout;
pub(crate) mod safety;
pub mod seatbelt;
mod session_rollout_init_error;
pub mod shell;
pub mod shell_snapshot;
pub mod skills;
pub mod spawn;
pub mod state_db;
pub mod state_db_bridge;
pub use codex_rollout::state_db;
mod thread_rollout_truncation;
mod tools;
pub mod turn_diff_tracker;
mod turn_metadata;
+63
View File
@@ -0,0 +1,63 @@
use crate::config::Config;
pub use codex_rollout::ARCHIVED_SESSIONS_SUBDIR;
pub use codex_rollout::INTERACTIVE_SESSION_SOURCES;
pub use codex_rollout::RolloutRecorder;
pub use codex_rollout::RolloutRecorderParams;
pub use codex_rollout::SESSIONS_SUBDIR;
pub use codex_rollout::SessionMeta;
pub use codex_rollout::append_thread_name;
pub use codex_rollout::find_archived_thread_path_by_id_str;
#[deprecated(note = "use find_thread_path_by_id_str")]
pub use codex_rollout::find_conversation_path_by_id_str;
pub use codex_rollout::find_thread_name_by_id;
pub use codex_rollout::find_thread_path_by_id_str;
pub use codex_rollout::find_thread_path_by_name_str;
pub use codex_rollout::rollout_date_parts;
impl codex_rollout::RolloutConfigView for Config {
fn codex_home(&self) -> &std::path::Path {
self.codex_home.as_path()
}
fn sqlite_home(&self) -> &std::path::Path {
self.sqlite_home.as_path()
}
fn cwd(&self) -> &std::path::Path {
self.cwd.as_path()
}
fn model_provider_id(&self) -> &str {
self.model_provider_id.as_str()
}
fn generate_memories(&self) -> bool {
self.memories.generate_memories
}
}
pub mod list {
pub use codex_rollout::list::*;
}
pub(crate) mod metadata {
pub(crate) use codex_rollout::metadata::builder_from_items;
}
pub mod policy {
pub use codex_rollout::policy::*;
}
pub mod recorder {
pub use codex_rollout::recorder::*;
}
pub mod session_index {
pub use codex_rollout::session_index::*;
}
pub(crate) use crate::session_rollout_init_error::map_session_init_error;
pub(crate) mod truncation {
pub(crate) use crate::thread_rollout_truncation::*;
}
+21
View File
@@ -0,0 +1,21 @@
use codex_rollout::state_db as rollout_state_db;
pub use codex_rollout::state_db::StateDbHandle;
pub use codex_rollout::state_db::apply_rollout_items;
pub use codex_rollout::state_db::find_rollout_path_by_id;
pub use codex_rollout::state_db::get_dynamic_tools;
pub use codex_rollout::state_db::list_thread_ids_db;
pub use codex_rollout::state_db::list_threads_db;
pub use codex_rollout::state_db::mark_thread_memory_mode_polluted;
pub use codex_rollout::state_db::normalize_cwd_for_state_db;
pub use codex_rollout::state_db::open_if_present;
pub use codex_rollout::state_db::persist_dynamic_tools;
pub use codex_rollout::state_db::read_repair_rollout_path;
pub use codex_rollout::state_db::reconcile_rollout;
pub use codex_rollout::state_db::touch_thread_updated_at;
pub use codex_state::LogEntry;
use crate::config::Config;
pub async fn get_state_db(config: &Config) -> Option<StateDbHandle> {
rollout_state_db::get_state_db(config).await
}
@@ -69,5 +69,5 @@ pub(crate) fn truncate_rollout_before_nth_user_message_from_start(
}
#[cfg(test)]
#[path = "truncation_tests.rs"]
#[path = "thread_rollout_truncation_tests.rs"]
mod tests;
+1
View File
@@ -0,0 +1 @@
pub mod path_utils;
+1
View File
@@ -0,0 +1 @@
pub use codex_utils_path::*;
+6
View File
@@ -0,0 +1,6 @@
load("//:defs.bzl", "codex_rust_crate")
codex_rust_crate(
name = "rollout",
crate_name = "codex_rollout",
)
+49
View File
@@ -0,0 +1,49 @@
[package]
edition.workspace = true
license.workspace = true
name = "codex-rollout"
version.workspace = true
[lib]
doctest = false
name = "codex_rollout"
path = "src/lib.rs"
[lints]
workspace = true
[dependencies]
anyhow = { workspace = true }
async-trait = { workspace = true }
chrono = { workspace = true, features = ["serde"] }
codex-file-search = { workspace = true }
codex-git-utils = { workspace = true }
codex-login = { workspace = true }
codex-otel = { workspace = true }
codex-protocol = { workspace = true }
codex-state = { workspace = true }
codex-utils-path = { workspace = true }
codex-utils-string = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
time = { workspace = true, features = [
"formatting",
"local-offset",
"macros",
"parsing",
] }
tokio = { workspace = true, features = [
"fs",
"io-util",
"macros",
"process",
"rt",
"sync",
"time",
] }
tracing = { workspace = true }
uuid = { workspace = true }
[dev-dependencies]
pretty_assertions = { workspace = true }
tempfile = { workspace = true }
+100
View File
@@ -0,0 +1,100 @@
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
pub trait RolloutConfigView {
fn codex_home(&self) -> &Path;
fn sqlite_home(&self) -> &Path;
fn cwd(&self) -> &Path;
fn model_provider_id(&self) -> &str;
fn generate_memories(&self) -> bool;
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RolloutConfig {
pub codex_home: PathBuf,
pub sqlite_home: PathBuf,
pub cwd: PathBuf,
pub model_provider_id: String,
pub generate_memories: bool,
}
pub type Config = RolloutConfig;
impl RolloutConfig {
pub fn from_view(view: &impl RolloutConfigView) -> Self {
Self {
codex_home: view.codex_home().to_path_buf(),
sqlite_home: view.sqlite_home().to_path_buf(),
cwd: view.cwd().to_path_buf(),
model_provider_id: view.model_provider_id().to_string(),
generate_memories: view.generate_memories(),
}
}
}
impl RolloutConfigView for RolloutConfig {
fn codex_home(&self) -> &Path {
self.codex_home.as_path()
}
fn sqlite_home(&self) -> &Path {
self.sqlite_home.as_path()
}
fn cwd(&self) -> &Path {
self.cwd.as_path()
}
fn model_provider_id(&self) -> &str {
self.model_provider_id.as_str()
}
fn generate_memories(&self) -> bool {
self.generate_memories
}
}
impl<T: RolloutConfigView + ?Sized> RolloutConfigView for &T {
fn codex_home(&self) -> &Path {
(*self).codex_home()
}
fn sqlite_home(&self) -> &Path {
(*self).sqlite_home()
}
fn cwd(&self) -> &Path {
(*self).cwd()
}
fn model_provider_id(&self) -> &str {
(*self).model_provider_id()
}
fn generate_memories(&self) -> bool {
(*self).generate_memories()
}
}
impl<T: RolloutConfigView + ?Sized> RolloutConfigView for Arc<T> {
fn codex_home(&self) -> &Path {
self.as_ref().codex_home()
}
fn sqlite_home(&self) -> &Path {
self.as_ref().sqlite_home()
}
fn cwd(&self) -> &Path {
self.as_ref().cwd()
}
fn model_provider_id(&self) -> &str {
self.as_ref().model_provider_id()
}
fn generate_memories(&self) -> bool {
self.as_ref().generate_memories()
}
}
@@ -1,9 +1,23 @@
//! Rollout module: persistence and discovery of session rollout files.
//! Rollout persistence and discovery for Codex session files.
use std::sync::LazyLock;
use codex_protocol::protocol::SessionSource;
pub mod config;
pub mod list;
pub mod metadata;
pub mod policy;
pub mod recorder;
pub mod session_index;
pub mod state_db;
pub(crate) mod default_client {
pub use codex_login::default_client::*;
}
pub(crate) use codex_protocol::protocol;
pub const SESSIONS_SUBDIR: &str = "sessions";
pub const ARCHIVED_SESSIONS_SUBDIR: &str = "archived_sessions";
pub static INTERACTIVE_SESSION_SOURCES: LazyLock<Vec<SessionSource>> = LazyLock::new(|| {
@@ -15,26 +29,22 @@ pub static INTERACTIVE_SESSION_SOURCES: LazyLock<Vec<SessionSource>> = LazyLock:
]
});
pub(crate) mod error;
pub mod list;
pub(crate) mod metadata;
pub(crate) mod policy;
pub mod recorder;
pub(crate) mod session_index;
pub(crate) mod truncation;
pub use codex_protocol::protocol::SessionMeta;
pub(crate) use error::map_session_init_error;
pub use config::RolloutConfig;
pub use config::RolloutConfigView;
pub use list::find_archived_thread_path_by_id_str;
pub use list::find_thread_path_by_id_str;
#[deprecated(note = "use find_thread_path_by_id_str")]
pub use list::find_thread_path_by_id_str as find_conversation_path_by_id_str;
pub use list::rollout_date_parts;
pub use policy::EventPersistenceMode;
pub use recorder::RolloutRecorder;
pub use recorder::RolloutRecorderParams;
pub use session_index::append_thread_name;
pub use session_index::find_thread_name_by_id;
pub use session_index::find_thread_names_by_ids;
pub use session_index::find_thread_path_by_name_str;
pub use state_db::StateDbHandle;
#[cfg(test)]
pub mod tests;
mod tests;
@@ -1,3 +1,5 @@
#![allow(warnings, clippy::all)]
use async_trait::async_trait;
use std::cmp::Reverse;
use std::ffi::OsStr;
@@ -111,16 +113,16 @@ pub enum ThreadSortKey {
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ThreadListLayout {
pub enum ThreadListLayout {
NestedByDate,
Flat,
}
pub(crate) struct ThreadListConfig<'a> {
pub(crate) allowed_sources: &'a [SessionSource],
pub(crate) model_providers: Option<&'a [String]>,
pub(crate) default_provider: &'a str,
pub(crate) layout: ThreadListLayout,
pub struct ThreadListConfig<'a> {
pub allowed_sources: &'a [SessionSource],
pub model_providers: Option<&'a [String]>,
pub default_provider: &'a str,
pub layout: ThreadListLayout,
}
/// Pagination cursor identifying a file by timestamp and UUID.
@@ -300,7 +302,7 @@ impl From<codex_state::Anchor> for Cursor {
/// can be supplied on the next call to resume after the last returned item, resilient to
/// concurrent new sessions being appended. Ordering is stable by the requested sort key
/// (timestamp desc, then UUID desc).
pub(crate) async fn get_threads(
pub async fn get_threads(
codex_home: &Path,
page_size: usize,
cursor: Option<&Cursor>,
@@ -325,7 +327,7 @@ pub(crate) async fn get_threads(
.await
}
pub(crate) async fn get_threads_in_root(
pub async fn get_threads_in_root(
root: PathBuf,
page_size: usize,
cursor: Option<&Cursor>,
@@ -1,7 +1,9 @@
use crate::config::Config;
use crate::rollout;
use crate::rollout::list::parse_timestamp_uuid_from_filename;
use crate::rollout::recorder::RolloutRecorder;
use crate::ARCHIVED_SESSIONS_SUBDIR;
use crate::SESSIONS_SUBDIR;
use crate::config::RolloutConfigView;
use crate::list;
use crate::list::parse_timestamp_uuid_from_filename;
use crate::recorder::RolloutRecorder;
use crate::state_db::normalize_cwd_for_state_db;
use chrono::DateTime;
use chrono::NaiveDateTime;
@@ -62,7 +64,7 @@ pub(crate) fn builder_from_session_meta(
Some(builder)
}
pub(crate) fn builder_from_items(
pub fn builder_from_items(
items: &[RolloutItem],
rollout_path: &Path,
) -> Option<ThreadMetadataBuilder> {
@@ -93,7 +95,7 @@ pub(crate) fn builder_from_items(
))
}
pub(crate) async fn extract_metadata_from_rollout(
pub async fn extract_metadata_from_rollout(
rollout_path: &Path,
default_provider: &str,
) -> anyhow::Result<ExtractionOutcome> {
@@ -131,7 +133,10 @@ pub(crate) async fn extract_metadata_from_rollout(
})
}
pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, config: &Config) {
pub(crate) async fn backfill_sessions(
runtime: &codex_state::StateRuntime,
config: &impl RolloutConfigView,
) {
let metric_client = codex_otel::metrics::global();
let timer = metric_client
.as_ref()
@@ -141,7 +146,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi
Err(err) => {
warn!(
"failed to read backfill state at {}: {err}",
config.codex_home.display()
config.codex_home().display()
);
BackfillState::default()
}
@@ -154,7 +159,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi
Err(err) => {
warn!(
"failed to claim backfill worker at {}: {err}",
config.codex_home.display()
config.codex_home().display()
);
return;
}
@@ -162,7 +167,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi
if !claimed {
info!(
"state db backfill already running at {}; skipping duplicate worker",
config.codex_home.display()
config.codex_home().display()
);
return;
}
@@ -171,7 +176,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi
Err(err) => {
warn!(
"failed to read claimed backfill state at {}: {err}",
config.codex_home.display()
config.codex_home().display()
);
BackfillState {
status: BackfillStatus::Running,
@@ -183,15 +188,15 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi
if let Err(err) = runtime.mark_backfill_running().await {
warn!(
"failed to mark backfill running at {}: {err}",
config.codex_home.display()
config.codex_home().display()
);
} else {
backfill_state.status = BackfillStatus::Running;
}
}
let sessions_root = config.codex_home.join(rollout::SESSIONS_SUBDIR);
let archived_root = config.codex_home.join(rollout::ARCHIVED_SESSIONS_SUBDIR);
let sessions_root = config.codex_home().join(SESSIONS_SUBDIR);
let archived_root = config.codex_home().join(ARCHIVED_SESSIONS_SUBDIR);
let mut rollout_paths: Vec<BackfillRolloutPath> = Vec::new();
for (root, archived) in [(sessions_root, false), (archived_root, true)] {
if !tokio::fs::try_exists(&root).await.unwrap_or(false) {
@@ -200,7 +205,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi
match collect_rollout_paths(&root).await {
Ok(paths) => {
rollout_paths.extend(paths.into_iter().map(|path| BackfillRolloutPath {
watermark: backfill_watermark_for_path(config.codex_home.as_path(), &path),
watermark: backfill_watermark_for_path(config.codex_home(), &path),
path,
archived,
}));
@@ -227,9 +232,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi
for batch in rollout_paths.chunks(BACKFILL_BATCH_SIZE) {
for rollout in batch {
stats.scanned = stats.scanned.saturating_add(1);
match extract_metadata_from_rollout(&rollout.path, config.model_provider_id.as_str())
.await
{
match extract_metadata_from_rollout(&rollout.path, config.model_provider_id()).await {
Ok(outcome) => {
if outcome.parse_errors > 0
&& let Some(ref metric_client) = metric_client
@@ -268,9 +271,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi
continue;
}
stats.upserted = stats.upserted.saturating_add(1);
if let Ok(meta_line) =
rollout::list::read_session_meta_line(&rollout.path).await
{
if let Ok(meta_line) = list::read_session_meta_line(&rollout.path).await {
if let Err(err) = runtime
.persist_dynamic_tools(
meta_line.meta.id,
@@ -308,7 +309,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi
{
warn!(
"failed to checkpoint backfill at {}: {err}",
config.codex_home.display()
config.codex_home().display()
);
} else {
last_watermark = Some(last_entry.watermark.clone());
@@ -321,7 +322,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi
{
warn!(
"failed to mark backfill complete at {}: {err}",
config.codex_home.display()
config.codex_home().display()
);
}
@@ -1,4 +1,7 @@
#![allow(warnings, clippy::all)]
use super::*;
use crate::config::RolloutConfig;
use chrono::DateTime;
use chrono::NaiveDateTime;
use chrono::Timelike;
@@ -21,6 +24,16 @@ use std::path::PathBuf;
use tempfile::tempdir;
use uuid::Uuid;
fn test_config(codex_home: PathBuf) -> RolloutConfig {
RolloutConfig {
sqlite_home: codex_home.clone(),
cwd: codex_home.clone(),
codex_home,
model_provider_id: "test-provider".to_string(),
generate_memories: true,
}
}
#[tokio::test]
async fn extract_metadata_from_rollout_uses_session_meta() {
let dir = tempdir().expect("tempdir");
@@ -197,9 +210,7 @@ async fn backfill_sessions_resumes_from_watermark_and_marks_complete() {
))
.await;
let mut config = crate::config::test_config();
config.codex_home = codex_home.clone();
config.model_provider_id = "test-provider".to_string();
let config = test_config(codex_home.clone());
backfill_sessions(runtime.as_ref(), &config).await;
let first_id = ThreadId::from_string(&first_uuid.to_string()).expect("first thread id");
@@ -267,9 +278,7 @@ async fn backfill_sessions_preserves_existing_git_branch_and_fills_missing_git_f
.await
.expect("existing metadata upsert");
let mut config = crate::config::test_config();
config.codex_home = codex_home.clone();
config.model_provider_id = "test-provider".to_string();
let config = test_config(codex_home.clone());
backfill_sessions(runtime.as_ref(), &config).await;
let persisted = runtime
@@ -304,9 +313,7 @@ async fn backfill_sessions_normalizes_cwd_before_upsert() {
.await
.expect("initialize runtime");
let mut config = crate::config::test_config();
config.codex_home = codex_home.clone();
config.model_provider_id = "test-provider".to_string();
let config = test_config(codex_home.clone());
backfill_sessions(runtime.as_ref(), &config).await;
let thread_id = ThreadId::from_string(&thread_uuid.to_string()).expect("thread id");
@@ -11,8 +11,7 @@ pub enum EventPersistenceMode {
/// Whether a rollout `item` should be persisted in rollout files for the
/// provided persistence `mode`.
#[inline]
pub(crate) fn is_persisted_response_item(item: &RolloutItem, mode: EventPersistenceMode) -> bool {
pub fn is_persisted_response_item(item: &RolloutItem, mode: EventPersistenceMode) -> bool {
match item {
RolloutItem::ResponseItem(item) => should_persist_response_item(item),
RolloutItem::EventMsg(ev) => should_persist_event_msg(ev, mode),
@@ -25,7 +24,7 @@ pub(crate) fn is_persisted_response_item(item: &RolloutItem, mode: EventPersiste
/// Whether a `ResponseItem` should be persisted in rollout files.
#[inline]
pub(crate) fn should_persist_response_item(item: &ResponseItem) -> bool {
pub fn should_persist_response_item(item: &ResponseItem) -> bool {
match item {
ResponseItem::Message { .. }
| ResponseItem::Reasoning { .. }
@@ -46,7 +45,7 @@ pub(crate) fn should_persist_response_item(item: &ResponseItem) -> bool {
/// Whether a `ResponseItem` should be persisted for the memories.
#[inline]
pub(crate) fn should_persist_response_item_for_memories(item: &ResponseItem) -> bool {
pub fn should_persist_response_item_for_memories(item: &ResponseItem) -> bool {
match item {
ResponseItem::Message { role, .. } => role != "developer",
ResponseItem::LocalShellCall { .. }
@@ -68,7 +67,7 @@ pub(crate) fn should_persist_response_item_for_memories(item: &ResponseItem) ->
/// Whether an `EventMsg` should be persisted in rollout files for the
/// provided persistence `mode`.
#[inline]
pub(crate) fn should_persist_event_msg(ev: &EventMsg, mode: EventPersistenceMode) -> bool {
pub fn should_persist_event_msg(ev: &EventMsg, mode: EventPersistenceMode) -> bool {
match mode {
EventPersistenceMode::Limited => should_persist_event_msg_limited(ev),
EventPersistenceMode::Extended => should_persist_event_msg_extended(ev),
@@ -11,6 +11,7 @@ use chrono::Utc;
use codex_protocol::ThreadId;
use codex_protocol::dynamic_tools::DynamicToolSpec;
use codex_protocol::models::BaseInstructions;
use codex_utils_string::truncate_middle_chars;
use serde_json::Value;
use time::OffsetDateTime;
use time::format_description::FormatItem;
@@ -39,9 +40,8 @@ use super::list::parse_timestamp_uuid_from_filename;
use super::metadata;
use super::policy::EventPersistenceMode;
use super::policy::is_persisted_response_item;
use crate::config::Config;
use crate::config::RolloutConfigView;
use crate::default_client::originator;
use crate::path_utils;
use crate::state_db;
use crate::state_db::StateDbHandle;
use codex_git_utils::collect_git_info;
@@ -56,8 +56,7 @@ use codex_protocol::protocol::SessionMetaLine;
use codex_protocol::protocol::SessionSource;
use codex_state::StateRuntime;
use codex_state::ThreadMetadataBuilder;
use codex_utils_output_truncation::TruncationPolicy;
use codex_utils_output_truncation::truncate_text;
use codex_utils_path as path_utils;
/// Records all [`ResponseItem`]s for a session and flushes them to disk after
/// every update.
@@ -146,9 +145,9 @@ fn sanitize_rollout_item_for_persistence(
match item {
RolloutItem::EventMsg(EventMsg::ExecCommandEnd(mut event)) => {
// Persist only a bounded aggregated summary of command output.
event.aggregated_output = truncate_text(
event.aggregated_output = truncate_middle_chars(
&event.aggregated_output,
TruncationPolicy::Bytes(PERSISTED_EXEC_AGGREGATED_OUTPUT_MAX_BYTES),
PERSISTED_EXEC_AGGREGATED_OUTPUT_MAX_BYTES,
);
// Drop unnecessary fields from rollout storage since aggregated_output is all we need.
event.stdout.clear();
@@ -164,7 +163,7 @@ impl RolloutRecorder {
/// List threads (rollout files) under the provided Codex home directory.
#[allow(clippy::too_many_arguments)]
pub async fn list_threads(
config: &Config,
config: &impl RolloutConfigView,
page_size: usize,
cursor: Option<&Cursor>,
sort_key: ThreadSortKey,
@@ -190,7 +189,7 @@ impl RolloutRecorder {
/// List archived threads (rollout files) under the archived sessions directory.
#[allow(clippy::too_many_arguments)]
pub async fn list_archived_threads(
config: &Config,
config: &impl RolloutConfigView,
page_size: usize,
cursor: Option<&Cursor>,
sort_key: ThreadSortKey,
@@ -215,7 +214,7 @@ impl RolloutRecorder {
#[allow(clippy::too_many_arguments)]
async fn list_threads_with_db_fallback(
config: &Config,
config: &impl RolloutConfigView,
page_size: usize,
cursor: Option<&Cursor>,
sort_key: ThreadSortKey,
@@ -225,7 +224,7 @@ impl RolloutRecorder {
archived: bool,
search_term: Option<&str>,
) -> std::io::Result<ThreadsPage> {
let codex_home = config.codex_home.as_path();
let codex_home = config.codex_home();
// Filesystem-first listing intentionally overfetches so we can repair stale/missing
// SQLite rollout paths before the final DB-backed page is returned.
let fs_page_size = page_size.saturating_mul(2).max(page_size);
@@ -299,7 +298,7 @@ impl RolloutRecorder {
/// Find the newest recorded thread path, optionally filtering to a matching cwd.
#[allow(clippy::too_many_arguments)]
pub async fn find_latest_thread_path(
config: &Config,
config: &impl RolloutConfigView,
page_size: usize,
cursor: Option<&Cursor>,
sort_key: ThreadSortKey,
@@ -308,7 +307,7 @@ impl RolloutRecorder {
default_provider: &str,
filter_cwd: Option<&Path>,
) -> std::io::Result<Option<PathBuf>> {
let codex_home = config.codex_home.as_path();
let codex_home = config.codex_home();
let state_db_ctx = state_db::get_state_db(config).await;
if state_db_ctx.is_some() {
let mut db_cursor = cursor.cloned();
@@ -369,7 +368,7 @@ impl RolloutRecorder {
///
/// For resumed sessions, this immediately opens the existing rollout file.
pub async fn new(
config: &Config,
config: &impl RolloutConfigView,
params: RolloutRecorderParams,
state_db_ctx: Option<StateDbHandle>,
state_builder: Option<ThreadMetadataBuilder>,
@@ -401,21 +400,21 @@ impl RolloutRecorder {
id: session_id,
forked_from_id,
timestamp,
cwd: config.cwd.clone(),
cwd: config.cwd().to_path_buf(),
originator: originator().value,
cli_version: env!("CARGO_PKG_VERSION").to_string(),
agent_nickname: source.get_nickname(),
agent_role: source.get_agent_role(),
agent_path: source.get_agent_path().map(Into::into),
source,
model_provider: Some(config.model_provider_id.clone()),
model_provider: Some(config.model_provider_id().to_string()),
base_instructions: Some(base_instructions),
dynamic_tools: if dynamic_tools.is_empty() {
None
} else {
Some(dynamic_tools)
},
memory_mode: (!config.memories.generate_memories)
memory_mode: (!config.generate_memories())
.then_some("disabled".to_string()),
};
@@ -445,7 +444,7 @@ impl RolloutRecorder {
};
// Clone the cwd for the spawned task to collect git info asynchronously
let cwd = config.cwd.clone();
let cwd = config.cwd().to_path_buf();
// A reasonably-sized bounded channel. If the buffer fills up the send
// future will yield, which is fine we only need to ensure we do not
@@ -463,8 +462,8 @@ impl RolloutRecorder {
rollout_path.clone(),
state_db_ctx.clone(),
state_builder,
config.model_provider_id.clone(),
config.memories.generate_memories,
config.model_provider_id().to_string(),
config.generate_memories(),
));
Ok(Self {
@@ -483,7 +482,7 @@ impl RolloutRecorder {
self.state_db.clone()
}
pub(crate) async fn record_items(&self, items: &[RolloutItem]) -> std::io::Result<()> {
pub async fn record_items(&self, items: &[RolloutItem]) -> std::io::Result<()> {
let mut filtered = Vec::new();
for item in items {
// Note that function calls may look a bit strange if they are
@@ -529,7 +528,7 @@ impl RolloutRecorder {
.map_err(|e| IoError::other(format!("failed waiting for rollout flush: {e}")))
}
pub(crate) async fn load_rollout_items(
pub async fn load_rollout_items(
path: &Path,
) -> std::io::Result<(Vec<RolloutItem>, Option<ThreadId>, usize)> {
trace!("Resuming rollout from {path:?}");
@@ -661,13 +660,13 @@ struct LogFileInfo {
}
fn precompute_log_file_info(
config: &Config,
config: &impl RolloutConfigView,
conversation_id: ThreadId,
) -> std::io::Result<LogFileInfo> {
// Resolve ~/.codex/sessions/YYYY/MM/DD path.
let timestamp = OffsetDateTime::now_local()
.map_err(|e| IoError::other(format!("failed to get local time: {e}")))?;
let mut dir = config.codex_home.clone();
let mut dir = config.codex_home().to_path_buf();
dir.push(SESSIONS_SUBDIR);
dir.push(timestamp.year().to_string());
dir.push(format!("{:02}", u8::from(timestamp.month())));
@@ -1,7 +1,8 @@
#![allow(warnings, clippy::all)]
use super::*;
use crate::config::ConfigBuilder;
use crate::config::RolloutConfig;
use chrono::TimeZone;
use codex_features::Feature;
use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig;
use codex_protocol::protocol::AgentMessageEvent;
use codex_protocol::protocol::AskForApproval;
@@ -19,6 +20,16 @@ use std::time::Duration;
use tempfile::TempDir;
use uuid::Uuid;
fn test_config(codex_home: &Path) -> RolloutConfig {
RolloutConfig {
codex_home: codex_home.to_path_buf(),
sqlite_home: codex_home.to_path_buf(),
cwd: codex_home.to_path_buf(),
model_provider_id: "test-provider".to_string(),
generate_memories: true,
}
}
fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result<PathBuf> {
let day_dir = root.join("sessions/2025/01/03");
fs::create_dir_all(&day_dir)?;
@@ -54,10 +65,7 @@ fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result<Path
#[tokio::test]
async fn recorder_materializes_only_after_explicit_persist() -> std::io::Result<()> {
let home = TempDir::new().expect("temp dir");
let config = ConfigBuilder::default()
.codex_home(home.path().to_path_buf())
.build()
.await?;
let config = test_config(home.path());
let thread_id = ThreadId::new();
let recorder = RolloutRecorder::new(
&config,
@@ -141,14 +149,7 @@ async fn recorder_materializes_only_after_explicit_persist() -> std::io::Result<
#[tokio::test]
async fn metadata_irrelevant_events_touch_state_db_updated_at() -> std::io::Result<()> {
let home = TempDir::new().expect("temp dir");
let mut config = ConfigBuilder::default()
.codex_home(home.path().to_path_buf())
.build()
.await?;
config
.features
.enable(Feature::Sqlite)
.expect("test config should allow sqlite");
let config = test_config(home.path());
let state_db = StateRuntime::init(home.path().to_path_buf(), config.model_provider_id.clone())
.await
@@ -229,14 +230,7 @@ async fn metadata_irrelevant_events_touch_state_db_updated_at() -> std::io::Resu
async fn metadata_irrelevant_events_fall_back_to_upsert_when_thread_missing() -> std::io::Result<()>
{
let home = TempDir::new().expect("temp dir");
let mut config = ConfigBuilder::default()
.codex_home(home.path().to_path_buf())
.build()
.await?;
config
.features
.enable(Feature::Sqlite)
.expect("test config should allow sqlite");
let config = test_config(home.path());
let state_db = StateRuntime::init(home.path().to_path_buf(), config.model_provider_id.clone())
.await
@@ -280,14 +274,7 @@ async fn metadata_irrelevant_events_fall_back_to_upsert_when_thread_missing() ->
#[tokio::test]
async fn list_threads_db_disabled_does_not_skip_paginated_items() -> std::io::Result<()> {
let home = TempDir::new().expect("temp dir");
let mut config = ConfigBuilder::default()
.codex_home(home.path().to_path_buf())
.build()
.await?;
config
.features
.disable(Feature::Sqlite)
.expect("test config should allow sqlite to be disabled");
let config = test_config(home.path());
let newest = write_session_file(home.path(), "2025-01-03T12-00-00", Uuid::from_u128(9001))?;
let middle = write_session_file(home.path(), "2025-01-02T12-00-00", Uuid::from_u128(9002))?;
@@ -328,14 +315,7 @@ async fn list_threads_db_disabled_does_not_skip_paginated_items() -> std::io::Re
#[tokio::test]
async fn list_threads_db_enabled_drops_missing_rollout_paths() -> std::io::Result<()> {
let home = TempDir::new().expect("temp dir");
let mut config = ConfigBuilder::default()
.codex_home(home.path().to_path_buf())
.build()
.await?;
config
.features
.enable(Feature::Sqlite)
.expect("test config should allow sqlite");
let config = test_config(home.path());
let uuid = Uuid::from_u128(9010);
let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id");
@@ -396,14 +376,7 @@ async fn list_threads_db_enabled_drops_missing_rollout_paths() -> std::io::Resul
#[tokio::test]
async fn list_threads_db_enabled_repairs_stale_rollout_paths() -> std::io::Result<()> {
let home = TempDir::new().expect("temp dir");
let mut config = ConfigBuilder::default()
.codex_home(home.path().to_path_buf())
.build()
.await?;
config
.features
.enable(Feature::Sqlite)
.expect("test config should allow sqlite");
let config = test_config(home.path());
let uuid = Uuid::from_u128(9011);
let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id");
@@ -1,3 +1,5 @@
#![allow(warnings, clippy::all)]
use super::*;
use pretty_assertions::assert_eq;
use std::collections::HashMap;
@@ -1,8 +1,8 @@
use crate::config::Config;
use crate::path_utils::normalize_for_path_comparison;
use crate::rollout::list::Cursor;
use crate::rollout::list::ThreadSortKey;
use crate::rollout::metadata;
use crate::config::RolloutConfig;
use crate::config::RolloutConfigView;
use crate::list::Cursor;
use crate::list::ThreadSortKey;
use crate::metadata;
use chrono::DateTime;
use chrono::NaiveDateTime;
use chrono::Timelike;
@@ -13,6 +13,7 @@ use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::SessionSource;
pub use codex_state::LogEntry;
use codex_state::ThreadMetadataBuilder;
use codex_utils_path::normalize_for_path_comparison;
use serde_json::Value;
use std::path::Path;
use std::path::PathBuf;
@@ -23,9 +24,9 @@ use uuid::Uuid;
/// Core-facing handle to the SQLite-backed state runtime.
pub type StateDbHandle = Arc<codex_state::StateRuntime>;
/// Initialize the state runtime for thread state persistence and backfill checks. To only be used
/// inside `core`. The initialization should not be done anywhere else.
pub(crate) async fn init(config: &Config) -> Option<StateDbHandle> {
/// Initialize the state runtime for thread state persistence and backfill checks.
pub async fn init(config: &impl RolloutConfigView) -> Option<StateDbHandle> {
let config = RolloutConfig::from_view(config);
let runtime = match codex_state::StateRuntime::init(
config.sqlite_home.clone(),
config.model_provider_id.clone(),
@@ -62,18 +63,18 @@ pub(crate) async fn init(config: &Config) -> Option<StateDbHandle> {
}
/// Get the DB if the feature is enabled and the DB exists.
pub async fn get_state_db(config: &Config) -> Option<StateDbHandle> {
let state_path = codex_state::state_db_path(config.sqlite_home.as_path());
pub async fn get_state_db(config: &impl RolloutConfigView) -> Option<StateDbHandle> {
let state_path = codex_state::state_db_path(config.sqlite_home());
if !tokio::fs::try_exists(&state_path).await.unwrap_or(false) {
return None;
}
let runtime = codex_state::StateRuntime::init(
config.sqlite_home.clone(),
config.model_provider_id.clone(),
config.sqlite_home().to_path_buf(),
config.model_provider_id().to_string(),
)
.await
.ok()?;
require_backfill_complete(runtime, config.sqlite_home.as_path()).await
require_backfill_complete(runtime, config.sqlite_home()).await
}
/// Open the state runtime when the SQLite file exists, without feature gating.
@@ -135,7 +136,7 @@ fn cursor_to_anchor(cursor: Option<&Cursor>) -> Option<codex_state::Anchor> {
Some(codex_state::Anchor { ts, id })
}
pub(crate) fn normalize_cwd_for_state_db(cwd: &Path) -> PathBuf {
pub fn normalize_cwd_for_state_db(cwd: &Path) -> PathBuf {
normalize_for_path_comparison(cwd).unwrap_or_else(|_| cwd.to_path_buf())
}
@@ -398,7 +399,7 @@ pub async fn reconcile_rollout(
);
return;
}
if let Ok(meta_line) = crate::rollout::list::read_session_meta_line(rollout_path).await {
if let Ok(meta_line) = crate::list::read_session_meta_line(rollout_path).await {
persist_dynamic_tools(
Some(ctx),
meta_line.meta.id,
@@ -463,7 +464,7 @@ pub async fn read_repair_rollout_path(
if !saw_existing_metadata {
warn!("state db discrepancy during read_repair_rollout_path: upsert_needed (slow path)");
}
let default_provider = crate::rollout::list::read_session_meta_line(rollout_path)
let default_provider = crate::list::read_session_meta_line(rollout_path)
.await
.ok()
.and_then(|meta| meta.meta.model_provider)
@@ -1,6 +1,13 @@
#![allow(warnings, clippy::all)]
use super::*;
use crate::rollout::list::parse_cursor;
use crate::list::parse_cursor;
use chrono::DateTime;
use chrono::NaiveDateTime;
use chrono::Timelike;
use chrono::Utc;
use pretty_assertions::assert_eq;
use uuid::Uuid;
#[test]
fn cursor_to_anchor_normalizes_timestamp_format() {
@@ -1,3 +1,4 @@
#![allow(warnings, clippy::all)]
#![allow(clippy::unwrap_used, clippy::expect_used)]
use std::ffi::OsStr;
@@ -17,14 +18,15 @@ use time::format_description::FormatItem;
use time::macros::format_description;
use uuid::Uuid;
use crate::rollout::INTERACTIVE_SESSION_SOURCES;
use crate::rollout::list::Cursor;
use crate::rollout::list::ThreadItem;
use crate::rollout::list::ThreadSortKey;
use crate::rollout::list::ThreadsPage;
use crate::rollout::list::get_threads;
use crate::rollout::list::read_head_for_summary;
use crate::rollout::rollout_date_parts;
use crate::INTERACTIVE_SESSION_SOURCES;
use crate::find_thread_path_by_id_str;
use crate::list::Cursor;
use crate::list::ThreadItem;
use crate::list::ThreadSortKey;
use crate::list::ThreadsPage;
use crate::list::get_threads;
use crate::list::read_head_for_summary;
use crate::rollout_date_parts;
use anyhow::Result;
use codex_protocol::ThreadId;
use codex_protocol::models::ContentItem;
@@ -229,7 +231,7 @@ async fn find_thread_path_falls_back_when_db_path_is_stale() {
));
insert_state_db_thread(home, thread_id, stale_db_path.as_path(), false).await;
let found = crate::rollout::find_thread_path_by_id_str(home, &uuid.to_string())
let found = find_thread_path_by_id_str(home, &uuid.to_string())
.await
.expect("lookup should succeed");
assert_eq!(found, Some(fs_rollout_path.clone()));
@@ -255,7 +257,7 @@ async fn find_thread_path_repairs_missing_db_row_after_filesystem_fallback() {
.await
.expect("backfill should be complete");
let found = crate::rollout::find_thread_path_by_id_str(home, &uuid.to_string())
let found = find_thread_path_by_id_str(home, &uuid.to_string())
.await
.expect("lookup should succeed");
assert_eq!(found, Some(fs_rollout_path.clone()));
+6
View File
@@ -0,0 +1,6 @@
load("//:defs.bzl", "codex_rust_crate")
codex_rust_crate(
name = "path-utils",
crate_name = "codex_utils_path",
)
+17
View File
@@ -0,0 +1,17 @@
[package]
edition.workspace = true
license.workspace = true
name = "codex-utils-path"
version.workspace = true
[lints]
workspace = true
[dependencies]
codex-utils-absolute-path = { workspace = true }
dunce = { workspace = true }
tempfile = { workspace = true }
[dev-dependencies]
pretty_assertions = { workspace = true }
tempfile = { workspace = true }
@@ -1,3 +1,7 @@
//! Path normalization, symlink resolution, and atomic writes shared across Codex crates.
pub mod env;
use codex_utils_absolute_path::AbsolutePathBuf;
use std::collections::HashSet;
use std::io;
@@ -5,8 +9,6 @@ use std::path::Path;
use std::path::PathBuf;
use tempfile::NamedTempFile;
use crate::env;
pub fn normalize_for_path_comparison(path: impl AsRef<Path>) -> std::io::Result<PathBuf> {
let canonical = path.as_ref().canonicalize()?;
Ok(normalize_for_wsl(canonical))