mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
feat: sqlite 1 (#10004)
Add a `.sqlite` database to be used to store rollout metatdata (and later logs) This PR is phase 1: * Add the database and the required infrastructure * Add a backfill of the database * Persist the newly created rollout both in files and in the DB * When we need to get metadata or a rollout, consider the `JSONL` as the source of truth but compare the results with the DB and show any errors
This commit is contained in:
jif-oai
@@ -154,6 +154,7 @@ use crate::protocol::WarningEvent;
|
||||
use crate::rollout::RolloutRecorder;
|
||||
use crate::rollout::RolloutRecorderParams;
|
||||
use crate::rollout::map_session_init_error;
|
||||
use crate::rollout::metadata;
|
||||
use crate::shell;
|
||||
use crate::shell_snapshot::ShellSnapshot;
|
||||
use crate::skills::SkillError;
|
||||
@@ -165,6 +166,7 @@ use crate::skills::collect_explicit_skill_mentions;
|
||||
use crate::state::ActiveTurn;
|
||||
use crate::state::SessionServices;
|
||||
use crate::state::SessionState;
|
||||
use crate::state_db;
|
||||
use crate::tasks::GhostSnapshotTask;
|
||||
use crate::tasks::ReviewTask;
|
||||
use crate::tasks::SessionTask;
|
||||
@@ -420,6 +422,10 @@ impl Codex {
|
||||
let state = self.session.state.lock().await;
|
||||
state.session_configuration.thread_config_snapshot()
|
||||
}
|
||||
|
||||
pub(crate) fn state_db(&self) -> Option<state_db::StateDbHandle> {
|
||||
self.session.state_db()
|
||||
}
|
||||
}
|
||||
|
||||
/// Context for an initialized model agent
|
||||
@@ -699,6 +705,13 @@ impl Session {
|
||||
RolloutRecorderParams::resume(resumed_history.rollout_path.clone()),
|
||||
),
|
||||
};
|
||||
let state_builder = match &initial_history {
|
||||
InitialHistory::Resumed(resumed) => metadata::builder_from_items(
|
||||
resumed.history.as_slice(),
|
||||
resumed.rollout_path.as_path(),
|
||||
),
|
||||
InitialHistory::New | InitialHistory::Forked(_) => None,
|
||||
};
|
||||
|
||||
// Kick off independent async setup tasks in parallel to reduce startup latency.
|
||||
//
|
||||
@@ -707,11 +720,17 @@ impl Session {
|
||||
// - load history metadata
|
||||
let rollout_fut = async {
|
||||
if config.ephemeral {
|
||||
Ok(None)
|
||||
Ok::<_, anyhow::Error>((None, None))
|
||||
} else {
|
||||
RolloutRecorder::new(&config, rollout_params)
|
||||
.await
|
||||
.map(Some)
|
||||
let state_db_ctx = state_db::init_if_enabled(&config, None).await;
|
||||
let rollout_recorder = RolloutRecorder::new(
|
||||
&config,
|
||||
rollout_params,
|
||||
state_db_ctx.clone(),
|
||||
state_builder.clone(),
|
||||
)
|
||||
.await?;
|
||||
Ok((Some(rollout_recorder), state_db_ctx))
|
||||
}
|
||||
};
|
||||
|
||||
@@ -731,14 +750,14 @@ impl Session {
|
||||
|
||||
// Join all independent futures.
|
||||
let (
|
||||
rollout_recorder,
|
||||
rollout_recorder_and_state_db,
|
||||
(history_log_id, history_entry_count),
|
||||
(auth, mcp_servers, auth_statuses),
|
||||
) = tokio::join!(rollout_fut, history_meta_fut, auth_and_mcp_fut);
|
||||
|
||||
let rollout_recorder = rollout_recorder.map_err(|e| {
|
||||
let (rollout_recorder, state_db_ctx) = rollout_recorder_and_state_db.map_err(|e| {
|
||||
error!("failed to initialize rollout recorder: {e:#}");
|
||||
anyhow::Error::from(e)
|
||||
e
|
||||
})?;
|
||||
let rollout_path = rollout_recorder
|
||||
.as_ref()
|
||||
@@ -842,6 +861,7 @@ impl Session {
|
||||
tool_approvals: Mutex::new(ApprovalStore::default()),
|
||||
skills_manager,
|
||||
agent_control,
|
||||
state_db: state_db_ctx.clone(),
|
||||
};
|
||||
|
||||
let sess = Arc::new(Session {
|
||||
@@ -914,6 +934,10 @@ impl Session {
|
||||
self.tx_event.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn state_db(&self) -> Option<state_db::StateDbHandle> {
|
||||
self.services.state_db.clone()
|
||||
}
|
||||
|
||||
/// Ensure all rollout writes are durably flushed.
|
||||
pub(crate) async fn flush_rollout(&self) {
|
||||
let recorder = {
|
||||
@@ -4580,6 +4604,7 @@ mod tests {
|
||||
tool_approvals: Mutex::new(ApprovalStore::default()),
|
||||
skills_manager,
|
||||
agent_control,
|
||||
state_db: None,
|
||||
};
|
||||
|
||||
let turn_context = Session::make_turn_context(
|
||||
@@ -4691,6 +4716,7 @@ mod tests {
|
||||
tool_approvals: Mutex::new(ApprovalStore::default()),
|
||||
skills_manager,
|
||||
agent_control,
|
||||
state_db: None,
|
||||
};
|
||||
|
||||
let turn_context = Arc::new(Session::make_turn_context(
|
||||
|
||||
@@ -12,6 +12,8 @@ use codex_protocol::protocol::SessionSource;
|
||||
use std::path::PathBuf;
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::state_db::StateDbHandle;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ThreadConfigSnapshot {
|
||||
pub model: String,
|
||||
@@ -64,6 +66,10 @@ impl CodexThread {
|
||||
self.rollout_path.clone()
|
||||
}
|
||||
|
||||
pub fn state_db(&self) -> Option<StateDbHandle> {
|
||||
self.codex.state_db()
|
||||
}
|
||||
|
||||
pub async fn config_snapshot(&self) -> ThreadConfigSnapshot {
|
||||
self.codex.thread_config_snapshot().await
|
||||
}
|
||||
|
||||
@@ -101,6 +101,8 @@ pub enum Feature {
|
||||
RemoteModels,
|
||||
/// Experimental shell snapshotting.
|
||||
ShellSnapshot,
|
||||
/// Persist rollout metadata to a local SQLite database.
|
||||
Sqlite,
|
||||
/// Append additional AGENTS.md guidance to user instructions.
|
||||
ChildAgentsMd,
|
||||
/// Enforce UTF8 output in Powershell.
|
||||
@@ -377,6 +379,12 @@ pub const FEATURES: &[FeatureSpec] = &[
|
||||
},
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
id: Feature::Sqlite,
|
||||
key: "sqlite",
|
||||
stage: Stage::UnderDevelopment,
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
id: Feature::ChildAgentsMd,
|
||||
key: "child_agents_md",
|
||||
|
||||
@@ -91,6 +91,7 @@ pub mod shell;
|
||||
pub mod shell_snapshot;
|
||||
pub mod skills;
|
||||
pub mod spawn;
|
||||
pub mod state_db;
|
||||
pub mod terminal;
|
||||
mod tools;
|
||||
pub mod turn_diff_tracker;
|
||||
|
||||
@@ -19,7 +19,9 @@ use uuid::Uuid;
|
||||
use super::ARCHIVED_SESSIONS_SUBDIR;
|
||||
use super::SESSIONS_SUBDIR;
|
||||
use crate::protocol::EventMsg;
|
||||
use crate::state_db;
|
||||
use codex_file_search as file_search;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::RolloutLine;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
@@ -794,7 +796,7 @@ async fn collect_rollout_day_files(
|
||||
Ok(day_files)
|
||||
}
|
||||
|
||||
fn parse_timestamp_uuid_from_filename(name: &str) -> Option<(OffsetDateTime, Uuid)> {
|
||||
pub(crate) fn parse_timestamp_uuid_from_filename(name: &str) -> Option<(OffsetDateTime, Uuid)> {
|
||||
// Expected: rollout-YYYY-MM-DDThh-mm-ss-<uuid>.jsonl
|
||||
let core = name.strip_prefix("rollout-")?.strip_suffix(".jsonl")?;
|
||||
|
||||
@@ -1093,11 +1095,39 @@ async fn find_thread_path_by_id_str_in_subdir(
|
||||
)
|
||||
.map_err(|e| io::Error::other(format!("file search failed: {e}")))?;
|
||||
|
||||
Ok(results
|
||||
let found = results
|
||||
.matches
|
||||
.into_iter()
|
||||
.next()
|
||||
.map(|m| root.join(m.path)))
|
||||
.map(|m| root.join(m.path));
|
||||
|
||||
// Checking if DB is at parity.
|
||||
// TODO(jif): sqlite migration phase 1
|
||||
let archived_only = match subdir {
|
||||
SESSIONS_SUBDIR => Some(false),
|
||||
ARCHIVED_SESSIONS_SUBDIR => Some(true),
|
||||
_ => None,
|
||||
};
|
||||
let state_db_ctx = state_db::open_if_present(codex_home, "").await;
|
||||
if let Some(state_db_ctx) = state_db_ctx.as_deref()
|
||||
&& let Ok(thread_id) = ThreadId::from_string(id_str)
|
||||
{
|
||||
let db_path = state_db::find_rollout_path_by_id(
|
||||
Some(state_db_ctx),
|
||||
thread_id,
|
||||
archived_only,
|
||||
"find_path_query",
|
||||
)
|
||||
.await;
|
||||
let canonical_path = found.as_deref();
|
||||
if db_path.as_deref() != canonical_path {
|
||||
tracing::warn!(
|
||||
"state db path mismatch for thread {thread_id:?}: canonical={canonical_path:?} db={db_path:?}"
|
||||
);
|
||||
state_db::record_discrepancy("find_thread_path_by_id_str_in_subdir", "path_mismatch");
|
||||
}
|
||||
}
|
||||
Ok(found)
|
||||
}
|
||||
|
||||
/// Locate a recorded thread rollout file by its UUID string using the existing
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
use crate::config::Config;
|
||||
use crate::rollout;
|
||||
use crate::rollout::list::parse_timestamp_uuid_from_filename;
|
||||
use crate::rollout::recorder::RolloutRecorder;
|
||||
use chrono::DateTime;
|
||||
use chrono::NaiveDateTime;
|
||||
use chrono::Timelike;
|
||||
use chrono::Utc;
|
||||
use codex_otel::OtelManager;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::SandboxPolicy;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_state::BackfillStats;
|
||||
use codex_state::DB_ERROR_METRIC;
|
||||
use codex_state::ExtractionOutcome;
|
||||
use codex_state::ThreadMetadataBuilder;
|
||||
use codex_state::apply_rollout_item;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use tracing::warn;
|
||||
|
||||
const ROLLOUT_PREFIX: &str = "rollout-";
|
||||
const ROLLOUT_SUFFIX: &str = ".jsonl";
|
||||
|
||||
pub(crate) fn builder_from_session_meta(
|
||||
session_meta: &SessionMetaLine,
|
||||
rollout_path: &Path,
|
||||
) -> Option<ThreadMetadataBuilder> {
|
||||
let created_at = parse_timestamp_to_utc(session_meta.meta.timestamp.as_str())?;
|
||||
let mut builder = ThreadMetadataBuilder::new(
|
||||
session_meta.meta.id,
|
||||
rollout_path.to_path_buf(),
|
||||
created_at,
|
||||
session_meta.meta.source.clone(),
|
||||
);
|
||||
builder.model_provider = session_meta.meta.model_provider.clone();
|
||||
builder.cwd = session_meta.meta.cwd.clone();
|
||||
builder.sandbox_policy = SandboxPolicy::ReadOnly;
|
||||
builder.approval_mode = AskForApproval::OnRequest;
|
||||
if let Some(git) = session_meta.git.as_ref() {
|
||||
builder.git_sha = git.commit_hash.clone();
|
||||
builder.git_branch = git.branch.clone();
|
||||
builder.git_origin_url = git.repository_url.clone();
|
||||
}
|
||||
Some(builder)
|
||||
}
|
||||
|
||||
pub(crate) fn builder_from_items(
|
||||
items: &[RolloutItem],
|
||||
rollout_path: &Path,
|
||||
) -> Option<ThreadMetadataBuilder> {
|
||||
if let Some(session_meta) = items.iter().find_map(|item| match item {
|
||||
RolloutItem::SessionMeta(meta_line) => Some(meta_line),
|
||||
RolloutItem::ResponseItem(_)
|
||||
| RolloutItem::Compacted(_)
|
||||
| RolloutItem::TurnContext(_)
|
||||
| RolloutItem::EventMsg(_) => None,
|
||||
}) && let Some(builder) = builder_from_session_meta(session_meta, rollout_path)
|
||||
{
|
||||
return Some(builder);
|
||||
}
|
||||
|
||||
let file_name = rollout_path.file_name()?.to_str()?;
|
||||
if !file_name.starts_with(ROLLOUT_PREFIX) || !file_name.ends_with(ROLLOUT_SUFFIX) {
|
||||
return None;
|
||||
}
|
||||
let (created_ts, uuid) = parse_timestamp_uuid_from_filename(file_name)?;
|
||||
let created_at =
|
||||
DateTime::<Utc>::from_timestamp(created_ts.unix_timestamp(), 0)?.with_nanosecond(0)?;
|
||||
let id = ThreadId::from_string(&uuid.to_string()).ok()?;
|
||||
Some(ThreadMetadataBuilder::new(
|
||||
id,
|
||||
rollout_path.to_path_buf(),
|
||||
created_at,
|
||||
SessionSource::default(),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn extract_metadata_from_rollout(
|
||||
rollout_path: &Path,
|
||||
default_provider: &str,
|
||||
otel: Option<&OtelManager>,
|
||||
) -> anyhow::Result<ExtractionOutcome> {
|
||||
let (items, _thread_id, parse_errors) =
|
||||
RolloutRecorder::load_rollout_items(rollout_path).await?;
|
||||
if items.is_empty() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"empty session file: {}",
|
||||
rollout_path.display()
|
||||
));
|
||||
}
|
||||
let builder = builder_from_items(items.as_slice(), rollout_path).ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"rollout missing metadata builder: {}",
|
||||
rollout_path.display()
|
||||
)
|
||||
})?;
|
||||
let mut metadata = builder.build(default_provider);
|
||||
for item in &items {
|
||||
apply_rollout_item(&mut metadata, item, default_provider);
|
||||
}
|
||||
if let Some(updated_at) = file_modified_time_utc(rollout_path).await {
|
||||
metadata.updated_at = updated_at;
|
||||
}
|
||||
if parse_errors > 0
|
||||
&& let Some(otel) = otel
|
||||
{
|
||||
otel.counter(
|
||||
DB_ERROR_METRIC,
|
||||
parse_errors as i64,
|
||||
&[("stage", "extract_metadata_from_rollout")],
|
||||
);
|
||||
}
|
||||
Ok(ExtractionOutcome {
|
||||
metadata,
|
||||
parse_errors,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn backfill_sessions(
|
||||
runtime: &codex_state::StateRuntime,
|
||||
config: &Config,
|
||||
otel: Option<&OtelManager>,
|
||||
) -> BackfillStats {
|
||||
let sessions_root = config.codex_home.join(rollout::SESSIONS_SUBDIR);
|
||||
let archived_root = config.codex_home.join(rollout::ARCHIVED_SESSIONS_SUBDIR);
|
||||
let mut rollout_paths: Vec<(PathBuf, bool)> = Vec::new();
|
||||
for (root, archived) in [(sessions_root, false), (archived_root, true)] {
|
||||
if !tokio::fs::try_exists(&root).await.unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
match collect_rollout_paths(&root).await {
|
||||
Ok(paths) => {
|
||||
rollout_paths.extend(paths.into_iter().map(|path| (path, archived)));
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"failed to collect rollout paths under {}: {err}",
|
||||
root.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut stats = BackfillStats {
|
||||
scanned: 0,
|
||||
upserted: 0,
|
||||
failed: 0,
|
||||
};
|
||||
for (path, archived) in rollout_paths {
|
||||
stats.scanned = stats.scanned.saturating_add(1);
|
||||
match extract_metadata_from_rollout(&path, config.model_provider_id.as_str(), otel).await {
|
||||
Ok(outcome) => {
|
||||
if outcome.parse_errors > 0
|
||||
&& let Some(otel) = otel
|
||||
{
|
||||
otel.counter(
|
||||
DB_ERROR_METRIC,
|
||||
outcome.parse_errors as i64,
|
||||
&[("stage", "backfill_sessions")],
|
||||
);
|
||||
}
|
||||
let mut metadata = outcome.metadata;
|
||||
if archived && metadata.archived_at.is_none() {
|
||||
let fallback_archived_at = metadata.updated_at;
|
||||
metadata.archived_at = file_modified_time_utc(&path)
|
||||
.await
|
||||
.or(Some(fallback_archived_at));
|
||||
}
|
||||
if let Err(err) = runtime.upsert_thread(&metadata).await {
|
||||
stats.failed = stats.failed.saturating_add(1);
|
||||
warn!("failed to upsert rollout {}: {err}", path.display());
|
||||
} else {
|
||||
stats.upserted = stats.upserted.saturating_add(1);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
stats.failed = stats.failed.saturating_add(1);
|
||||
warn!("failed to extract rollout {}: {err}", path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
stats
|
||||
}
|
||||
|
||||
async fn file_modified_time_utc(path: &Path) -> Option<DateTime<Utc>> {
|
||||
let modified = tokio::fs::metadata(path).await.ok()?.modified().ok()?;
|
||||
let updated_at: DateTime<Utc> = modified.into();
|
||||
updated_at.with_nanosecond(0)
|
||||
}
|
||||
|
||||
fn parse_timestamp_to_utc(ts: &str) -> Option<DateTime<Utc>> {
|
||||
const FILENAME_TS_FORMAT: &str = "%Y-%m-%dT%H-%M-%S";
|
||||
if let Ok(naive) = NaiveDateTime::parse_from_str(ts, FILENAME_TS_FORMAT) {
|
||||
let dt = DateTime::<Utc>::from_naive_utc_and_offset(naive, Utc);
|
||||
return dt.with_nanosecond(0);
|
||||
}
|
||||
if let Ok(dt) = DateTime::parse_from_rfc3339(ts) {
|
||||
return dt.with_timezone(&Utc).with_nanosecond(0);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
async fn collect_rollout_paths(root: &Path) -> std::io::Result<Vec<PathBuf>> {
|
||||
let mut stack = vec![root.to_path_buf()];
|
||||
let mut paths = Vec::new();
|
||||
while let Some(dir) = stack.pop() {
|
||||
let mut read_dir = match tokio::fs::read_dir(&dir).await {
|
||||
Ok(read_dir) => read_dir,
|
||||
Err(err) => {
|
||||
warn!("failed to read directory {}: {err}", dir.display());
|
||||
continue;
|
||||
}
|
||||
};
|
||||
while let Some(entry) = read_dir.next_entry().await? {
|
||||
let path = entry.path();
|
||||
let file_type = entry.file_type().await?;
|
||||
if file_type.is_dir() {
|
||||
stack.push(path);
|
||||
continue;
|
||||
}
|
||||
if !file_type.is_file() {
|
||||
continue;
|
||||
}
|
||||
let file_name = entry.file_name();
|
||||
let Some(name) = file_name.to_str() else {
|
||||
continue;
|
||||
};
|
||||
if name.starts_with(ROLLOUT_PREFIX) && name.ends_with(ROLLOUT_SUFFIX) {
|
||||
paths.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(paths)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::DateTime;
|
||||
use chrono::NaiveDateTime;
|
||||
use chrono::Timelike;
|
||||
use chrono::Utc;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::CompactedItem;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::RolloutLine;
|
||||
use codex_protocol::protocol::SessionMeta;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_state::ThreadMetadataBuilder;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use tempfile::tempdir;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[tokio::test]
|
||||
async fn extract_metadata_from_rollout_uses_session_meta() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let uuid = Uuid::new_v4();
|
||||
let id = ThreadId::from_string(&uuid.to_string()).expect("thread id");
|
||||
let path = dir
|
||||
.path()
|
||||
.join(format!("rollout-2026-01-27T12-34-56-{uuid}.jsonl"));
|
||||
|
||||
let session_meta = SessionMeta {
|
||||
id,
|
||||
forked_from_id: None,
|
||||
timestamp: "2026-01-27T12:34:56Z".to_string(),
|
||||
cwd: dir.path().to_path_buf(),
|
||||
originator: "cli".to_string(),
|
||||
cli_version: "0.0.0".to_string(),
|
||||
source: SessionSource::default(),
|
||||
model_provider: Some("openai".to_string()),
|
||||
base_instructions: None,
|
||||
};
|
||||
let session_meta_line = SessionMetaLine {
|
||||
meta: session_meta,
|
||||
git: None,
|
||||
};
|
||||
let rollout_line = RolloutLine {
|
||||
timestamp: "2026-01-27T12:34:56Z".to_string(),
|
||||
item: RolloutItem::SessionMeta(session_meta_line.clone()),
|
||||
};
|
||||
let json = serde_json::to_string(&rollout_line).expect("rollout json");
|
||||
let mut file = File::create(&path).expect("create rollout");
|
||||
writeln!(file, "{json}").expect("write rollout");
|
||||
|
||||
let outcome = extract_metadata_from_rollout(&path, "openai", None)
|
||||
.await
|
||||
.expect("extract");
|
||||
|
||||
let builder =
|
||||
builder_from_session_meta(&session_meta_line, path.as_path()).expect("builder");
|
||||
let mut expected = builder.build("openai");
|
||||
apply_rollout_item(&mut expected, &rollout_line.item, "openai");
|
||||
expected.updated_at = file_modified_time_utc(&path).await.expect("mtime");
|
||||
|
||||
assert_eq!(outcome.metadata, expected);
|
||||
assert_eq!(outcome.parse_errors, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builder_from_items_falls_back_to_filename() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let uuid = Uuid::new_v4();
|
||||
let path = dir
|
||||
.path()
|
||||
.join(format!("rollout-2026-01-27T12-34-56-{uuid}.jsonl"));
|
||||
let items = vec![RolloutItem::Compacted(CompactedItem {
|
||||
message: "noop".to_string(),
|
||||
replacement_history: None,
|
||||
})];
|
||||
|
||||
let builder = builder_from_items(items.as_slice(), path.as_path()).expect("builder");
|
||||
let naive = NaiveDateTime::parse_from_str("2026-01-27T12-34-56", "%Y-%m-%dT%H-%M-%S")
|
||||
.expect("timestamp");
|
||||
let created_at = DateTime::<Utc>::from_naive_utc_and_offset(naive, Utc)
|
||||
.with_nanosecond(0)
|
||||
.expect("nanosecond");
|
||||
let expected = ThreadMetadataBuilder::new(
|
||||
ThreadId::from_string(&uuid.to_string()).expect("thread id"),
|
||||
path,
|
||||
created_at,
|
||||
SessionSource::default(),
|
||||
);
|
||||
|
||||
assert_eq!(builder, expected);
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ pub const INTERACTIVE_SESSION_SOURCES: &[SessionSource] =
|
||||
|
||||
pub(crate) mod error;
|
||||
pub mod list;
|
||||
pub(crate) mod metadata;
|
||||
pub(crate) mod policy;
|
||||
pub mod recorder;
|
||||
pub(crate) mod truncation;
|
||||
|
||||
@@ -28,11 +28,14 @@ use super::list::ThreadSortKey;
|
||||
use super::list::ThreadsPage;
|
||||
use super::list::get_threads;
|
||||
use super::list::get_threads_in_root;
|
||||
use super::metadata;
|
||||
use super::policy::is_persisted_response_item;
|
||||
use crate::config::Config;
|
||||
use crate::default_client::originator;
|
||||
use crate::git_info::collect_git_info;
|
||||
use crate::path_utils;
|
||||
use crate::state_db;
|
||||
use crate::state_db::StateDbHandle;
|
||||
use codex_protocol::protocol::InitialHistory;
|
||||
use codex_protocol::protocol::ResumedHistory;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
@@ -40,6 +43,7 @@ use codex_protocol::protocol::RolloutLine;
|
||||
use codex_protocol::protocol::SessionMeta;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_state::ThreadMetadataBuilder;
|
||||
|
||||
/// Records all [`ResponseItem`]s for a session and flushes them to disk after
|
||||
/// every update.
|
||||
@@ -54,6 +58,7 @@ use codex_protocol::protocol::SessionSource;
|
||||
pub struct RolloutRecorder {
|
||||
tx: Sender<RolloutCmd>,
|
||||
pub(crate) rollout_path: PathBuf,
|
||||
state_db: Option<StateDbHandle>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -111,7 +116,8 @@ impl RolloutRecorder {
|
||||
model_providers: Option<&[String]>,
|
||||
default_provider: &str,
|
||||
) -> std::io::Result<ThreadsPage> {
|
||||
get_threads(
|
||||
let stage = "list_threads";
|
||||
let page = get_threads(
|
||||
codex_home,
|
||||
page_size,
|
||||
cursor,
|
||||
@@ -120,7 +126,34 @@ impl RolloutRecorder {
|
||||
model_providers,
|
||||
default_provider,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// TODO(jif): drop after sqlite migration phase 1
|
||||
let state_db_ctx = state_db::open_if_present(codex_home, default_provider).await;
|
||||
if let Some(db_ids) = state_db::list_thread_ids_db(
|
||||
state_db_ctx.as_deref(),
|
||||
codex_home,
|
||||
page_size,
|
||||
cursor,
|
||||
sort_key,
|
||||
allowed_sources,
|
||||
model_providers,
|
||||
false,
|
||||
stage,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if page.items.len() != db_ids.len() {
|
||||
state_db::record_discrepancy(stage, "bad_len");
|
||||
return Ok(page);
|
||||
}
|
||||
for (id, item) in db_ids.iter().zip(page.items.iter()) {
|
||||
if !item.path.display().to_string().contains(&id.to_string()) {
|
||||
state_db::record_discrepancy(stage, "bad_id");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(page)
|
||||
}
|
||||
|
||||
/// List archived threads (rollout files) under the archived sessions directory.
|
||||
@@ -133,8 +166,9 @@ impl RolloutRecorder {
|
||||
model_providers: Option<&[String]>,
|
||||
default_provider: &str,
|
||||
) -> std::io::Result<ThreadsPage> {
|
||||
let stage = "list_archived_threads";
|
||||
let root = codex_home.join(ARCHIVED_SESSIONS_SUBDIR);
|
||||
get_threads_in_root(
|
||||
let page = get_threads_in_root(
|
||||
root,
|
||||
page_size,
|
||||
cursor,
|
||||
@@ -146,7 +180,34 @@ impl RolloutRecorder {
|
||||
layout: ThreadListLayout::Flat,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
// TODO(jif): drop after sqlite migration phase 1
|
||||
let state_db_ctx = state_db::open_if_present(codex_home, default_provider).await;
|
||||
if let Some(db_ids) = state_db::list_thread_ids_db(
|
||||
state_db_ctx.as_deref(),
|
||||
codex_home,
|
||||
page_size,
|
||||
cursor,
|
||||
sort_key,
|
||||
allowed_sources,
|
||||
model_providers,
|
||||
true,
|
||||
stage,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if page.items.len() != db_ids.len() {
|
||||
state_db::record_discrepancy(stage, "bad_len");
|
||||
return Ok(page);
|
||||
}
|
||||
for (id, item) in db_ids.iter().zip(page.items.iter()) {
|
||||
if !item.path.display().to_string().contains(&id.to_string()) {
|
||||
state_db::record_discrepancy(stage, "bad_id");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(page)
|
||||
}
|
||||
|
||||
/// Find the newest recorded thread path, optionally filtering to a matching cwd.
|
||||
@@ -186,7 +247,12 @@ impl RolloutRecorder {
|
||||
/// Attempt to create a new [`RolloutRecorder`]. If the sessions directory
|
||||
/// cannot be created or the rollout file cannot be opened we return the
|
||||
/// error so the caller can decide whether to disable persistence.
|
||||
pub async fn new(config: &Config, params: RolloutRecorderParams) -> std::io::Result<Self> {
|
||||
pub async fn new(
|
||||
config: &Config,
|
||||
params: RolloutRecorderParams,
|
||||
state_db_ctx: Option<StateDbHandle>,
|
||||
state_builder: Option<ThreadMetadataBuilder>,
|
||||
) -> std::io::Result<Self> {
|
||||
let (file, rollout_path, meta) = match params {
|
||||
RolloutRecorderParams::Create {
|
||||
conversation_id,
|
||||
@@ -246,9 +312,30 @@ impl RolloutRecorder {
|
||||
// Spawn a Tokio task that owns the file handle and performs async
|
||||
// writes. Using `tokio::fs::File` keeps everything on the async I/O
|
||||
// driver instead of blocking the runtime.
|
||||
tokio::task::spawn(rollout_writer(file, rx, meta, cwd));
|
||||
tokio::task::spawn(rollout_writer(
|
||||
file,
|
||||
rx,
|
||||
meta,
|
||||
cwd,
|
||||
rollout_path.clone(),
|
||||
state_db_ctx.clone(),
|
||||
state_builder,
|
||||
config.model_provider_id.clone(),
|
||||
));
|
||||
|
||||
Ok(Self { tx, rollout_path })
|
||||
Ok(Self {
|
||||
tx,
|
||||
rollout_path,
|
||||
state_db: state_db_ctx,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn rollout_path(&self) -> &Path {
|
||||
self.rollout_path.as_path()
|
||||
}
|
||||
|
||||
pub fn state_db(&self) -> Option<StateDbHandle> {
|
||||
self.state_db.clone()
|
||||
}
|
||||
|
||||
pub(crate) async fn record_items(&self, items: &[RolloutItem]) -> std::io::Result<()> {
|
||||
@@ -281,7 +368,9 @@ impl RolloutRecorder {
|
||||
.map_err(|e| IoError::other(format!("failed waiting for rollout flush: {e}")))
|
||||
}
|
||||
|
||||
pub async fn get_rollout_history(path: &Path) -> std::io::Result<InitialHistory> {
|
||||
pub(crate) async fn load_rollout_items(
|
||||
path: &Path,
|
||||
) -> std::io::Result<(Vec<RolloutItem>, Option<ThreadId>, usize)> {
|
||||
info!("Resuming rollout from {path:?}");
|
||||
let text = tokio::fs::read_to_string(path).await?;
|
||||
if text.trim().is_empty() {
|
||||
@@ -290,6 +379,7 @@ impl RolloutRecorder {
|
||||
|
||||
let mut items: Vec<RolloutItem> = Vec::new();
|
||||
let mut thread_id: Option<ThreadId> = None;
|
||||
let mut parse_errors = 0usize;
|
||||
for line in text.lines() {
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
@@ -298,6 +388,7 @@ impl RolloutRecorder {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
warn!("failed to parse line as JSON: {line:?}, error: {e}");
|
||||
parse_errors = parse_errors.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -328,15 +419,22 @@ impl RolloutRecorder {
|
||||
},
|
||||
Err(e) => {
|
||||
warn!("failed to parse rollout line: {v:?}, error: {e}");
|
||||
parse_errors = parse_errors.saturating_add(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
"Resumed rollout with {} items, thread ID: {:?}",
|
||||
"Resumed rollout with {} items, thread ID: {:?}, parse errors: {}",
|
||||
items.len(),
|
||||
thread_id
|
||||
thread_id,
|
||||
parse_errors,
|
||||
);
|
||||
Ok((items, thread_id, parse_errors))
|
||||
}
|
||||
|
||||
pub async fn get_rollout_history(path: &Path) -> std::io::Result<InitialHistory> {
|
||||
let (items, thread_id, _parse_errors) = Self::load_rollout_items(path).await?;
|
||||
let conversation_id = thread_id
|
||||
.ok_or_else(|| IoError::other("failed to parse thread ID from rollout file"))?;
|
||||
|
||||
@@ -417,13 +515,21 @@ fn create_log_file(config: &Config, conversation_id: ThreadId) -> std::io::Resul
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn rollout_writer(
|
||||
file: tokio::fs::File,
|
||||
mut rx: mpsc::Receiver<RolloutCmd>,
|
||||
mut meta: Option<SessionMeta>,
|
||||
cwd: std::path::PathBuf,
|
||||
rollout_path: PathBuf,
|
||||
state_db_ctx: Option<StateDbHandle>,
|
||||
mut state_builder: Option<ThreadMetadataBuilder>,
|
||||
default_provider: String,
|
||||
) -> std::io::Result<()> {
|
||||
let mut writer = JsonlWriter { file };
|
||||
if let Some(builder) = state_builder.as_mut() {
|
||||
builder.rollout_path = rollout_path.clone();
|
||||
}
|
||||
|
||||
// If we have a meta, collect git info asynchronously and write meta first
|
||||
if let Some(session_meta) = meta.take() {
|
||||
@@ -432,22 +538,50 @@ async fn rollout_writer(
|
||||
meta: session_meta,
|
||||
git: git_info,
|
||||
};
|
||||
if state_db_ctx.is_some() {
|
||||
state_builder =
|
||||
metadata::builder_from_session_meta(&session_meta_line, rollout_path.as_path());
|
||||
}
|
||||
|
||||
// Write the SessionMeta as the first item in the file, wrapped in a rollout line
|
||||
writer
|
||||
.write_rollout_item(RolloutItem::SessionMeta(session_meta_line))
|
||||
.await?;
|
||||
let rollout_item = RolloutItem::SessionMeta(session_meta_line);
|
||||
writer.write_rollout_item(&rollout_item).await?;
|
||||
state_db::reconcile_rollout(
|
||||
state_db_ctx.as_deref(),
|
||||
rollout_path.as_path(),
|
||||
default_provider.as_str(),
|
||||
state_builder.as_ref(),
|
||||
std::slice::from_ref(&rollout_item),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
// Process rollout commands
|
||||
while let Some(cmd) = rx.recv().await {
|
||||
match cmd {
|
||||
RolloutCmd::AddItems(items) => {
|
||||
let mut persisted_items = Vec::new();
|
||||
for item in items {
|
||||
if is_persisted_response_item(&item) {
|
||||
writer.write_rollout_item(item).await?;
|
||||
writer.write_rollout_item(&item).await?;
|
||||
persisted_items.push(item);
|
||||
}
|
||||
}
|
||||
if persisted_items.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Some(builder) = state_builder.as_mut() {
|
||||
builder.rollout_path = rollout_path.clone();
|
||||
}
|
||||
state_db::apply_rollout_items(
|
||||
state_db_ctx.as_deref(),
|
||||
rollout_path.as_path(),
|
||||
default_provider.as_str(),
|
||||
state_builder.as_ref(),
|
||||
persisted_items.as_slice(),
|
||||
"rollout_writer",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
RolloutCmd::Flush { ack } => {
|
||||
// Ensure underlying file is flushed and then ack.
|
||||
@@ -470,8 +604,15 @@ struct JsonlWriter {
|
||||
file: tokio::fs::File,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct RolloutLineRef<'a> {
|
||||
timestamp: String,
|
||||
#[serde(flatten)]
|
||||
item: &'a RolloutItem,
|
||||
}
|
||||
|
||||
impl JsonlWriter {
|
||||
async fn write_rollout_item(&mut self, rollout_item: RolloutItem) -> std::io::Result<()> {
|
||||
async fn write_rollout_item(&mut self, rollout_item: &RolloutItem) -> std::io::Result<()> {
|
||||
let timestamp_format: &[FormatItem] = format_description!(
|
||||
"[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
|
||||
);
|
||||
@@ -479,7 +620,7 @@ impl JsonlWriter {
|
||||
.format(timestamp_format)
|
||||
.map_err(|e| IoError::other(format!("failed to format timestamp: {e}")))?;
|
||||
|
||||
let line = RolloutLine {
|
||||
let line = RolloutLineRef {
|
||||
timestamp,
|
||||
item: rollout_item,
|
||||
};
|
||||
|
||||
@@ -7,6 +7,7 @@ use crate::exec_policy::ExecPolicyManager;
|
||||
use crate::mcp_connection_manager::McpConnectionManager;
|
||||
use crate::models_manager::manager::ModelsManager;
|
||||
use crate::skills::SkillsManager;
|
||||
use crate::state_db::StateDbHandle;
|
||||
use crate::tools::sandboxing::ApprovalStore;
|
||||
use crate::unified_exec::UnifiedExecProcessManager;
|
||||
use crate::user_notification::UserNotifier;
|
||||
@@ -30,4 +31,5 @@ pub(crate) struct SessionServices {
|
||||
pub(crate) tool_approvals: Mutex<ApprovalStore>,
|
||||
pub(crate) skills_manager: Arc<SkillsManager>,
|
||||
pub(crate) agent_control: AgentControl,
|
||||
pub(crate) state_db: Option<StateDbHandle>,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
use crate::config::Config;
|
||||
use crate::features::Feature;
|
||||
use crate::rollout::list::Cursor;
|
||||
use crate::rollout::list::ThreadSortKey;
|
||||
use crate::rollout::metadata;
|
||||
use chrono::DateTime;
|
||||
use chrono::NaiveDateTime;
|
||||
use chrono::Timelike;
|
||||
use chrono::Utc;
|
||||
use codex_otel::OtelManager;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_state::DB_METRIC_BACKFILL;
|
||||
use codex_state::STATE_DB_FILENAME;
|
||||
use codex_state::ThreadMetadataBuilder;
|
||||
use serde_json::Value;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Core-facing handle to the optional SQLite-backed state runtime.
|
||||
pub type StateDbHandle = Arc<codex_state::StateRuntime>;
|
||||
|
||||
/// Initialize the state runtime when the `sqlite` feature flag is enabled.
|
||||
pub async fn init_if_enabled(config: &Config, otel: Option<&OtelManager>) -> Option<StateDbHandle> {
|
||||
let state_path = config.codex_home.join(STATE_DB_FILENAME);
|
||||
if !config.features.enabled(Feature::Sqlite) {
|
||||
// We delete the file on best effort basis to maintain retro-compatibility in the future.
|
||||
let wal_path = state_path.with_extension("sqlite-wal");
|
||||
let shm_path = state_path.with_extension("sqlite-shm");
|
||||
for path in [state_path.as_path(), wal_path.as_path(), shm_path.as_path()] {
|
||||
tokio::fs::remove_file(path).await.ok();
|
||||
}
|
||||
return None;
|
||||
}
|
||||
let existed = tokio::fs::try_exists(&state_path).await.unwrap_or(false);
|
||||
let runtime = match codex_state::StateRuntime::init(
|
||||
config.codex_home.clone(),
|
||||
config.model_provider_id.clone(),
|
||||
otel.cloned(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(runtime) => runtime,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"failed to initialize state runtime at {}: {err}",
|
||||
config.codex_home.display()
|
||||
);
|
||||
if let Some(otel) = otel {
|
||||
otel.counter("codex.db.init", 1, &[("status", "init_error")]);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
};
|
||||
if !existed {
|
||||
let stats = metadata::backfill_sessions(runtime.as_ref(), config, otel).await;
|
||||
info!(
|
||||
"state db backfill scanned={}, upserted={}, failed={}",
|
||||
stats.scanned, stats.upserted, stats.failed
|
||||
);
|
||||
if let Some(otel) = otel {
|
||||
otel.counter(
|
||||
DB_METRIC_BACKFILL,
|
||||
stats.upserted as i64,
|
||||
&[("status", "upserted")],
|
||||
);
|
||||
otel.counter(
|
||||
DB_METRIC_BACKFILL,
|
||||
stats.failed as i64,
|
||||
&[("status", "failed")],
|
||||
);
|
||||
}
|
||||
}
|
||||
Some(runtime)
|
||||
}
|
||||
|
||||
/// Open the state runtime when the SQLite file exists, without feature gating.
|
||||
///
|
||||
/// This is used for parity checks during the SQLite migration phase.
|
||||
pub async fn open_if_present(codex_home: &Path, default_provider: &str) -> Option<StateDbHandle> {
|
||||
let db_path = codex_home.join(STATE_DB_FILENAME);
|
||||
if !tokio::fs::try_exists(&db_path).await.unwrap_or(false) {
|
||||
return None;
|
||||
}
|
||||
let runtime = codex_state::StateRuntime::init(
|
||||
codex_home.to_path_buf(),
|
||||
default_provider.to_string(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.ok()?;
|
||||
Some(runtime)
|
||||
}
|
||||
|
||||
fn cursor_to_anchor(cursor: Option<&Cursor>) -> Option<codex_state::Anchor> {
|
||||
let cursor = cursor?;
|
||||
let value = serde_json::to_value(cursor).ok()?;
|
||||
let cursor_str = value.as_str()?;
|
||||
let (ts_str, id_str) = cursor_str.split_once('|')?;
|
||||
if id_str.contains('|') {
|
||||
return None;
|
||||
}
|
||||
let id = Uuid::parse_str(id_str).ok()?;
|
||||
let ts = if let Ok(naive) = NaiveDateTime::parse_from_str(ts_str, "%Y-%m-%dT%H-%M-%S") {
|
||||
DateTime::<Utc>::from_naive_utc_and_offset(naive, Utc)
|
||||
} else if let Ok(dt) = DateTime::parse_from_rfc3339(ts_str) {
|
||||
dt.with_timezone(&Utc)
|
||||
} else {
|
||||
return None;
|
||||
}
|
||||
.with_nanosecond(0)?;
|
||||
Some(codex_state::Anchor { ts, id })
|
||||
}
|
||||
|
||||
/// List thread ids from SQLite for parity checks without rollout scanning.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn list_thread_ids_db(
|
||||
context: Option<&codex_state::StateRuntime>,
|
||||
codex_home: &Path,
|
||||
page_size: usize,
|
||||
cursor: Option<&Cursor>,
|
||||
sort_key: ThreadSortKey,
|
||||
allowed_sources: &[SessionSource],
|
||||
model_providers: Option<&[String]>,
|
||||
archived_only: bool,
|
||||
stage: &str,
|
||||
) -> Option<Vec<ThreadId>> {
|
||||
let ctx = context?;
|
||||
if ctx.codex_home() != codex_home {
|
||||
warn!(
|
||||
"state db codex_home mismatch: expected {}, got {}",
|
||||
ctx.codex_home().display(),
|
||||
codex_home.display()
|
||||
);
|
||||
}
|
||||
|
||||
let anchor = cursor_to_anchor(cursor);
|
||||
let allowed_sources: Vec<String> = allowed_sources
|
||||
.iter()
|
||||
.map(|value| match serde_json::to_value(value) {
|
||||
Ok(Value::String(s)) => s,
|
||||
Ok(other) => other.to_string(),
|
||||
Err(_) => String::new(),
|
||||
})
|
||||
.collect();
|
||||
let model_providers = model_providers.map(<[String]>::to_vec);
|
||||
match ctx
|
||||
.list_thread_ids(
|
||||
page_size,
|
||||
anchor.as_ref(),
|
||||
match sort_key {
|
||||
ThreadSortKey::CreatedAt => codex_state::SortKey::CreatedAt,
|
||||
ThreadSortKey::UpdatedAt => codex_state::SortKey::UpdatedAt,
|
||||
},
|
||||
allowed_sources.as_slice(),
|
||||
model_providers.as_deref(),
|
||||
archived_only,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(ids) => Some(ids),
|
||||
Err(err) => {
|
||||
warn!("state db list_thread_ids failed during {stage}: {err}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Look up the rollout path for a thread id using SQLite.
|
||||
pub async fn find_rollout_path_by_id(
|
||||
context: Option<&codex_state::StateRuntime>,
|
||||
thread_id: ThreadId,
|
||||
archived_only: Option<bool>,
|
||||
stage: &str,
|
||||
) -> Option<PathBuf> {
|
||||
let ctx = context?;
|
||||
ctx.find_rollout_path_by_id(thread_id, archived_only)
|
||||
.await
|
||||
.unwrap_or_else(|err| {
|
||||
warn!("state db find_rollout_path_by_id failed during {stage}: {err}");
|
||||
None
|
||||
})
|
||||
}
|
||||
|
||||
/// Reconcile rollout items into SQLite, falling back to scanning the rollout file.
|
||||
pub async fn reconcile_rollout(
|
||||
context: Option<&codex_state::StateRuntime>,
|
||||
rollout_path: &Path,
|
||||
default_provider: &str,
|
||||
builder: Option<&ThreadMetadataBuilder>,
|
||||
items: &[RolloutItem],
|
||||
) {
|
||||
let Some(ctx) = context else {
|
||||
return;
|
||||
};
|
||||
if builder.is_some() || !items.is_empty() {
|
||||
apply_rollout_items(
|
||||
Some(ctx),
|
||||
rollout_path,
|
||||
default_provider,
|
||||
builder,
|
||||
items,
|
||||
"reconcile_rollout",
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
let outcome =
|
||||
match metadata::extract_metadata_from_rollout(rollout_path, default_provider, None).await {
|
||||
Ok(outcome) => outcome,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"state db reconcile_rollout extraction failed {}: {err}",
|
||||
rollout_path.display()
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
if let Err(err) = ctx.upsert_thread(&outcome.metadata).await {
|
||||
warn!(
|
||||
"state db reconcile_rollout upsert failed {}: {err}",
|
||||
rollout_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply rollout items incrementally to SQLite.
|
||||
pub async fn apply_rollout_items(
|
||||
context: Option<&codex_state::StateRuntime>,
|
||||
rollout_path: &Path,
|
||||
_default_provider: &str,
|
||||
builder: Option<&ThreadMetadataBuilder>,
|
||||
items: &[RolloutItem],
|
||||
stage: &str,
|
||||
) {
|
||||
let Some(ctx) = context else {
|
||||
return;
|
||||
};
|
||||
let mut builder = match builder {
|
||||
Some(builder) => builder.clone(),
|
||||
None => match metadata::builder_from_items(items, rollout_path) {
|
||||
Some(builder) => builder,
|
||||
None => {
|
||||
warn!(
|
||||
"state db apply_rollout_items missing builder during {stage}: {}",
|
||||
rollout_path.display()
|
||||
);
|
||||
record_discrepancy(stage, "missing_builder");
|
||||
return;
|
||||
}
|
||||
},
|
||||
};
|
||||
builder.rollout_path = rollout_path.to_path_buf();
|
||||
if let Err(err) = ctx.apply_rollout_items(&builder, items, None).await {
|
||||
warn!(
|
||||
"state db apply_rollout_items failed during {stage} for {}: {err}",
|
||||
rollout_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a state discrepancy metric with a stage and reason tag.
|
||||
pub fn record_discrepancy(stage: &str, reason: &str) {
|
||||
// We access the global metric because the call sites might not have access to the broader
|
||||
// OtelManager.
|
||||
if let Some(metric) = codex_otel::metrics::global() {
|
||||
let _ = metric.counter(
|
||||
"codex.db.discrepancy",
|
||||
1,
|
||||
&[("stage", stage), ("reason", reason)],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::rollout::list::parse_cursor;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn cursor_to_anchor_normalizes_timestamp_format() {
|
||||
let uuid = Uuid::new_v4();
|
||||
let ts_str = "2026-01-27T12-34-56";
|
||||
let token = format!("{ts_str}|{uuid}");
|
||||
let cursor = parse_cursor(token.as_str()).expect("cursor should parse");
|
||||
let anchor = cursor_to_anchor(Some(&cursor)).expect("anchor should parse");
|
||||
|
||||
let naive =
|
||||
NaiveDateTime::parse_from_str(ts_str, "%Y-%m-%dT%H-%M-%S").expect("ts should parse");
|
||||
let expected_ts = DateTime::<Utc>::from_naive_utc_and_offset(naive, Utc)
|
||||
.with_nanosecond(0)
|
||||
.expect("nanosecond");
|
||||
|
||||
assert_eq!(anchor.id, uuid);
|
||||
assert_eq!(anchor.ts, expected_ts);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user