agent/codex
2
0
Fork 0
mirror of https://github.com/pchuan98/codex.git synced 2026-07-01 00:31:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: dedicated goal DB (#23300)

Browse Source 
## Why

Thread goals are moving toward extension-owned runtime behavior, but
their persisted state was still stored in the shared state database.
This makes the goal store harder to isolate and keeps future storage
splits tied to ad hoc runtime plumbing.

This PR gives goals their own SQLite database while keeping the existing
`StateRuntime` entry point. The goal is to make this the pattern for
adding more dedicated runtime databases later.

This also reduce load on existing DB and reduce contention

## Limitation
Thread preview from goal is not supported anymore. I'm looking into this
[EDIT]: solved

## What changed

- Added a dedicated `goals_1.sqlite` database with its own
`goals_migrations` directory.
- Moved `thread_goals` creation into the goals DB migration set.
- Dropped the old `thread_goals` table from the main state DB with a
normal state migration. There is intentionally no backfill for existing
goal rows.
- Changed `GoalStore` to be backed only by the goals DB pool.
- Removed the old goal-write side effect that filled empty
`threads.preview` values from the goal objective.
- Added shared runtime DB path metadata so startup, telemetry, `codex
doctor`, and repair handling can include future DBs without bespoke path
lists.
- Updated Bazel compile data so the new goals migration directory is
available to `sqlx::migrate!`.

## Verification

- `cargo check --tests -p codex-state -p codex-cli -p codex-core -p
codex-app-server`
- `just fix -p codex-state`
- `just fix -p codex-cli`
- `just fix -p codex-app-server`
This commit is contained in:
jif-oai
2026-05-19 11:11:41 +02:00
committed by GitHub
parent 826b2182ed
commit ba57aab13a
18 changed files with 330 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/state/src/runtime.rs
+101 -32
View File
@@ -5,6 +5,7 @@ use crate::AgentJobItemCreateParams;
use crate::AgentJobItemStatus;
use crate::AgentJobProgress;
use crate::AgentJobStatus;
use crate::GOALS_DB_FILENAME;
use crate::LOGS_DB_FILENAME;
use crate::LogEntry;
use crate::LogQuery;
@@ -15,6 +16,7 @@ use crate::ThreadMetadata;
use crate::ThreadMetadataBuilder;
use crate::ThreadsPage;
use crate::apply_rollout_item;
use crate::migrations::runtime_goals_migrator;
use crate::migrations::runtime_logs_migrator;
use crate::migrations::runtime_state_migrator;
use crate::model::AgentJobRow;
@@ -80,6 +82,53 @@ pub use threads::ThreadFilterOptions;
const LOG_PARTITION_SIZE_LIMIT_BYTES: i64 = 10 * 1024 * 1024;
const LOG_PARTITION_ROW_LIMIT: i64 = 1_000;
#[derive(Clone, Copy)]
struct RuntimeDbSpec {
label: &'static str,
filename: &'static str,
kind: DbKind,
open_phase: &'static str,
migrate_phase: &'static str,
}
impl RuntimeDbSpec {
fn path(self, codex_home: &Path) -> PathBuf {
codex_home.join(self.filename)
}
}
const STATE_DB: RuntimeDbSpec = RuntimeDbSpec {
label: "state DB",
filename: STATE_DB_FILENAME,
kind: DbKind::State,
open_phase: "open_state",
migrate_phase: "migrate_state",
};
const LOGS_DB: RuntimeDbSpec = RuntimeDbSpec {
label: "log DB",
filename: LOGS_DB_FILENAME,
kind: DbKind::Logs,
open_phase: "open_logs",
migrate_phase: "migrate_logs",
};
const GOALS_DB: RuntimeDbSpec = RuntimeDbSpec {
label: "goals DB",
filename: GOALS_DB_FILENAME,
kind: DbKind::Goals,
open_phase: "open_goals",
migrate_phase: "migrate_goals",
};
const RUNTIME_DBS: [RuntimeDbSpec; 3] = [STATE_DB, LOGS_DB, GOALS_DB];
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RuntimeDbPath {
pub label: &'static str,
pub path: PathBuf,
}
#[derive(Clone)]
pub struct StateRuntime {
codex_home: PathBuf,
@@ -122,8 +171,10 @@ impl StateRuntime {
tokio::fs::create_dir_all(&codex_home).await?;
let state_migrator = runtime_state_migrator();
let logs_migrator = runtime_logs_migrator();
let state_path = state_db_path(codex_home.as_path());
let logs_path = logs_db_path(codex_home.as_path());
let goals_migrator = runtime_goals_migrator();
let state_path = STATE_DB.path(codex_home.as_path());
let logs_path = LOGS_DB.path(codex_home.as_path());
let goals_path = GOALS_DB.path(codex_home.as_path());
let pool = match open_state_sqlite(&state_path, &state_migrator, telemetry_override).await {
Ok(db) => Arc::new(db),
Err(err) => {
@@ -139,6 +190,14 @@ impl StateRuntime {
return Err(err);
}
};
let goals_pool =
match open_goals_sqlite(&goals_path, &goals_migrator, telemetry_override).await {
Ok(db) => Arc::new(db),
Err(err) => {
warn!("failed to open goals db at {}: {err}", goals_path.display());
return Err(err);
}
};
let started = Instant::now();
let backfill_state_result = ensure_backfill_state_row_in_pool(pool.as_ref()).await;
crate::telemetry::record_init_result(
@@ -165,7 +224,7 @@ impl StateRuntime {
let thread_updated_at_millis = thread_updated_at_millis_result?;
let thread_updated_at_millis = thread_updated_at_millis.unwrap_or(0);
let runtime = Arc::new(Self {
thread_goals: GoalStore::new(Arc::clone(&pool)),
thread_goals: GoalStore::new(Arc::clone(&goals_pool)),
pool,
logs_pool,
codex_home,
@@ -209,15 +268,7 @@ async fn open_state_sqlite(
// New state DBs should use incremental auto-vacuum, but retrofitting an
// existing DB requires a full VACUUM. Do not attempt that during process
// startup: it is maintenance work that can contend with foreground writers.
open_sqlite(
path,
migrator,
DbKind::State,
"open_state",
"migrate_state",
telemetry_override,
)
.await
open_sqlite(path, migrator, STATE_DB, telemetry_override).await
}
async fn open_logs_sqlite(
@@ -225,23 +276,21 @@ async fn open_logs_sqlite(
migrator: &Migrator,
telemetry_override: Option<&dyn DbTelemetry>,
) -> anyhow::Result<SqlitePool> {
open_sqlite(
path,
migrator,
DbKind::Logs,
"open_logs",
"migrate_logs",
telemetry_override,
)
.await
open_sqlite(path, migrator, LOGS_DB, telemetry_override).await
}
async fn open_goals_sqlite(
path: &Path,
migrator: &Migrator,
telemetry_override: Option<&dyn DbTelemetry>,
) -> anyhow::Result<SqlitePool> {
open_sqlite(path, migrator, GOALS_DB, telemetry_override).await
}
async fn open_sqlite(
path: &Path,
migrator: &Migrator,
db: DbKind,
open_phase: &'static str,
migrate_phase: &'static str,
spec: RuntimeDbSpec,
telemetry_override: Option<&dyn DbTelemetry>,
) -> anyhow::Result<SqlitePool> {
let options = base_sqlite_options(path).auto_vacuum(SqliteAutoVacuum::Incremental);
@@ -253,8 +302,8 @@ async fn open_sqlite(
.map_err(anyhow::Error::from);
crate::telemetry::record_init_result(
telemetry_override,
db,
open_phase,
spec.kind,
spec.open_phase,
started.elapsed(),
&pool_result,
);
@@ -263,8 +312,8 @@ async fn open_sqlite(
let migrate_result = migrator.run(&pool).await.map_err(anyhow::Error::from);
crate::telemetry::record_init_result(
telemetry_override,
db,
migrate_phase,
spec.kind,
spec.migrate_phase,
started.elapsed(),
&migrate_result,
);
@@ -291,19 +340,37 @@ ON CONFLICT(id) DO NOTHING
}
pub fn state_db_filename() -> String {
STATE_DB_FILENAME.to_string()
STATE_DB.filename.to_string()
}
pub fn state_db_path(codex_home: &Path) -> PathBuf {
codex_home.join(state_db_filename())
STATE_DB.path(codex_home)
}
pub fn logs_db_filename() -> String {
LOGS_DB_FILENAME.to_string()
LOGS_DB.filename.to_string()
}
pub fn logs_db_path(codex_home: &Path) -> PathBuf {
codex_home.join(logs_db_filename())
LOGS_DB.path(codex_home)
}
pub fn goals_db_filename() -> String {
GOALS_DB.filename.to_string()
}
pub fn goals_db_path(codex_home: &Path) -> PathBuf {
GOALS_DB.path(codex_home)
}
pub fn runtime_db_paths(codex_home: &Path) -> Vec<RuntimeDbPath> {
RUNTIME_DBS
.iter()
.map(|spec| RuntimeDbPath {
label: spec.label,
path: spec.path(codex_home),
})
.collect()
}
/// Run SQLite's built-in integrity check against an existing database file.
@@ -510,6 +577,8 @@ mod tests {
"migrate_state",
"open_logs",
"migrate_logs",
"open_goals",
"migrate_goals",
"ensure_backfill_state",
"post_init_query",
]