Move memory state to a dedicated SQLite DB (#24591)

## Summary

Generated memory rows and their stage-one/stage-two job state currently
live in `state_5.sqlite` alongside thread metadata. That makes memory
cleanup and regeneration share the main state schema even though those
rows are memory-pipeline data and can be rebuilt independently from the
durable thread records.

This PR moves the memory-owned tables into a dedicated
`memories_1.sqlite` runtime database while keeping thread metadata in
`state_5.sqlite`.

## Changes

- Adds a separate memories DB runtime, migrator, path helpers, telemetry
kind, and Bazel compile data for `state/memory_migrations`.
- Introduces `MemoryStore` behind `StateRuntime::memories()` and moves
memory table/job operations onto that store.
- Drops the old memory tables from the state DB and recreates their
schema in `state/memory_migrations/0001_memories.sql`.
- Updates memory startup, citation usage tracking, rollout pollution
handling, `debug clear-memories`, and app-server `memory/reset` to
operate through the memories DB.
- Preserves cross-DB behavior by hydrating thread metadata from the
state DB when selecting visible memory outputs and checking stage-one
staleness.

## Verification

- Added/updated `codex-state` tests for deleted-thread memory visibility
and already-polluted phase-two enqueue behavior.
- Updated `debug clear-memories`, app-server `memory/reset`, and
memories startup tests to seed and assert memory rows through
`memories_1.sqlite`.
This commit is contained in:
jif-oai
2026-05-26 20:07:25 +02:00
committed by GitHub
parent 823381e867
commit aad59a0916
22 changed files with 1014 additions and 280 deletions
+32
View File
@@ -702,6 +702,7 @@ fn state_summary(check: &DoctorCheck) -> String {
"state DB integrity",
"log DB integrity",
"goals DB integrity",
"memories DB integrity",
]
.into_iter()
.all(|label| detail::detail_value(check, label).is_some_and(|value| value == "ok"));
@@ -1363,6 +1364,37 @@ Run codex doctor without --summary for detailed diagnostics.
);
}
#[test]
fn render_human_report_includes_memories_db_in_state_health_summary() {
let report = DoctorReport {
schema_version: 1,
generated_at: "0s since unix epoch".to_string(),
overall_status: CheckStatus::Ok,
codex_version: "0.0.0".to_string(),
checks: vec![
DoctorCheck::new(
"state.paths",
"state",
CheckStatus::Ok,
"state paths inspectable",
)
.detail("state DB: /tmp/state.sqlite")
.detail("state DB integrity: ok")
.detail("log DB: /tmp/logs.sqlite")
.detail("log DB integrity: ok")
.detail("goals DB: /tmp/goals.sqlite")
.detail("goals DB integrity: ok")
.detail("memories DB: /tmp/memories.sqlite")
.detail("memories DB integrity: ok"),
],
};
let rendered = render_human_report(&report, detailed_no_color_unicode_options());
assert!(rendered.contains("✓ state databases healthy"));
assert!(rendered.contains("memories DB /tmp/memories.sqlite · integrity ok"));
}
#[test]
fn render_human_report_supports_ascii_output() {
let rendered = render_human_report(
+3
View File
@@ -413,6 +413,7 @@ fn state_details(parsed: &[ParsedDetail]) -> Vec<HumanDetail> {
push_database_row(&mut out, parsed, "state DB");
push_database_row(&mut out, parsed, "log DB");
push_database_row(&mut out, parsed, "goals DB");
push_database_row(&mut out, parsed, "memories DB");
for (source, label) in [
("active rollout files", "active rollouts"),
@@ -440,6 +441,8 @@ fn state_details(parsed: &[ParsedDetail]) -> Vec<HumanDetail> {
"state DB integrity",
"log DB integrity",
"goals DB integrity",
"memories DB",
"memories DB integrity",
"active rollout files",
"archived rollout files",
],
+7 -13
View File
@@ -27,7 +27,7 @@ use codex_responses_api_proxy::Args as ResponsesApiProxyArgs;
use codex_rollout_trace::REDUCED_STATE_FILE_NAME;
use codex_rollout_trace::replay_bundle;
use codex_state::StateRuntime;
use codex_state::state_db_path;
use codex_state::memories_db_path;
use codex_tui::AppExitInfo;
use codex_tui::Cli as TuiCli;
use codex_tui::ExitReason;
@@ -1751,22 +1751,16 @@ async fn run_debug_clear_memories_command(
.build()
.await?;
let state_path = state_db_path(config.sqlite_home.as_path());
let mut cleared_state_db = false;
if tokio::fs::try_exists(&state_path).await? {
let state_db =
StateRuntime::init(config.sqlite_home.clone(), config.model_provider_id.clone())
.await?;
state_db.clear_memory_data().await?;
cleared_state_db = true;
}
let memories_path = memories_db_path(config.sqlite_home.as_path());
let cleared_memories_db =
StateRuntime::clear_memory_data_in_sqlite_home(config.sqlite_home.as_path()).await?;
clear_memory_roots_contents(&config.codex_home).await?;
let mut message = if cleared_state_db {
format!("Cleared memory state from {}.", state_path.display())
let mut message = if cleared_memories_db {
format!("Cleared memory state from {}.", memories_path.display())
} else {
format!("No state db found at {}.", state_path.display())
format!("No memories db found at {}.", memories_path.display())
};
message.push_str(&format!(
" Cleared memory directories under {}.",