feat: house-keeping memories 3 (#20005)

Move stuff in memories, no behavioural change expected
This commit is contained in:
jif-oai
2026-04-28 18:13:35 +02:00
committed by GitHub
Unverified
parent 0156b1e61f
commit 5b7d6f5c4f
11 changed files with 127 additions and 106 deletions
+4 -5
View File
@@ -6,15 +6,12 @@ use chrono::Utc;
use std::path::Path;
use tracing::warn;
const FILENAME_TS_FORMAT: &str = "%Y-%m-%dT%H-%M-%S";
const EXTENSION_RESOURCE_RETENTION_DAYS: i64 = 7;
pub async fn prune_old_extension_resources(memory_root: &Path) {
prune_old_extension_resources_with_now(memory_root, Utc::now()).await
}
async fn prune_old_extension_resources_with_now(memory_root: &Path, now: DateTime<Utc>) {
let cutoff = now - Duration::days(EXTENSION_RESOURCE_RETENTION_DAYS);
let cutoff = now - Duration::days(crate::extension_resources::RETENTION_DAYS);
let extensions_root = memory_extensions_root(memory_root);
let mut extensions = match tokio::fs::read_dir(&extensions_root).await {
Ok(extensions) => extensions,
@@ -92,7 +89,9 @@ async fn prune_old_extension_resources_with_now(memory_root: &Path, now: DateTim
fn resource_timestamp(file_name: &str) -> Option<DateTime<Utc>> {
let timestamp = file_name.get(..19)?;
let naive = NaiveDateTime::parse_from_str(timestamp, FILENAME_TS_FORMAT).ok()?;
let naive =
NaiveDateTime::parse_from_str(timestamp, crate::extension_resources::FILENAME_TS_FORMAT)
.ok()?;
Some(DateTime::from_naive_utc_and_offset(naive, Utc))
}
@@ -19,8 +19,11 @@ async fn prunes_only_old_resources_from_extensions_with_instructions() {
.expect("write chronicle instructions");
let now = DateTime::from_naive_utc_and_offset(
NaiveDateTime::parse_from_str("2026-04-14T12-00-00", FILENAME_TS_FORMAT)
.expect("parse now"),
NaiveDateTime::parse_from_str(
"2026-04-14T12-00-00",
crate::extension_resources::FILENAME_TS_FORMAT,
)
.expect("parse now"),
Utc,
);
let old_file = chronicle_resources.join("2026-04-06T11-59-59-abcd-10min-old.md");
+1 -3
View File
@@ -6,8 +6,6 @@ use codex_protocol::protocol::RateLimitWindow;
use tracing::info;
use tracing::warn;
const CODEX_LIMIT_ID: &str = "codex";
pub(crate) async fn rate_limits_ok(auth_manager: &AuthManager, config: &Config) -> bool {
rate_limits_check(auth_manager, config)
.await
@@ -32,7 +30,7 @@ async fn rate_limits_check(auth_manager: &AuthManager, config: &Config) -> Optio
let snapshot = snapshots
.iter()
.find(|s| s.limit_id.as_deref() == Some(CODEX_LIMIT_ID))
.find(|s| s.limit_id.as_deref() == Some(crate::guard_limits::CODEX_LIMIT_ID))
.or_else(|| snapshots.first())?;
let min_remaining_percent = config.memories.min_rate_limit_remaining_percent;
+1 -1
View File
@@ -6,7 +6,7 @@ fn snapshot(
secondary_used_percent: Option<f64>,
) -> RateLimitSnapshot {
RateLimitSnapshot {
limit_id: Some(CODEX_LIMIT_ID.to_string()),
limit_id: Some(crate::guard_limits::CODEX_LIMIT_ID.to_string()),
limit_name: None,
primary: primary_used_percent.map(window),
secondary: secondary_used_percent.map(window),
+77 -14
View File
@@ -29,20 +29,6 @@ pub use storage::rebuild_raw_memories_file_from_memories;
pub use storage::rollout_summary_file_stem;
pub use storage::sync_rollout_summaries_from_memories;
/// Prompt used for phase 1 extraction.
pub const STAGE_ONE_PROMPT: &str = include_str!("../templates/memories/stage_one_system.md");
/// Fallback stage-1 rollout truncation limit (tokens) when model metadata
/// does not include a valid context window.
pub const DEFAULT_STAGE_ONE_ROLLOUT_TOKEN_LIMIT: usize = 150_000;
/// Portion of the model effective input window reserved for the stage-1
/// rollout input.
///
/// Keeping this below 100% leaves room for system instructions, prompt framing,
/// and model output.
pub const STAGE_ONE_CONTEXT_WINDOW_PERCENT: i64 = 70;
#[cfg(test)]
mod startup_tests;
@@ -52,6 +38,83 @@ mod artifacts {
pub(super) const RAW_MEMORIES_FILENAME: &str = "raw_memories.md";
}
mod extension_resources {
pub(super) const FILENAME_TS_FORMAT: &str = "%Y-%m-%dT%H-%M-%S";
pub(super) const RETENTION_DAYS: i64 = 7;
}
mod guard_limits {
pub(super) const CODEX_LIMIT_ID: &str = "codex";
}
mod prompt_blocks {
pub(super) const EXTENSIONS_FOLDER_STRUCTURE: &str = r#"
Memory extensions (under {{ memory_extensions_root }}/):
- <extension_name>/instructions.md
- Source-specific guidance for interpreting additional memory signals. If an
extension folder exists, you must read its instructions.md to determine how to use this memory
source.
If the user has any memory extensions, you MUST read the instructions for each extension to
determine how to use the memory source. If the workspace diff shows deleted extension resource files,
remove stale memories derived only from those resources. If it has no extension folders, continue
with the standard memory inputs only.
"#;
pub(super) const EXTENSIONS_PRIMARY_INPUTS: &str = r#"
Optional source-specific inputs:
Under `{{ memory_extensions_root }}/`:
- `<extension_name>/instructions.md`
- If extension folders exist, read each instructions.md first and follow it when interpreting
that extension's memory source.
If the workspace diff shows deleted memory extension resources, use that extension-specific deletion
signal to remove stale memories derived only from those resources.
"#;
}
mod stage_one {
pub(super) const MODEL: &str = "gpt-5.4-mini";
pub(super) const REASONING_EFFORT: codex_protocol::openai_models::ReasoningEffort =
codex_protocol::openai_models::ReasoningEffort::Low;
pub(super) const CONCURRENCY_LIMIT: usize = 8;
pub(super) const JOB_LEASE_SECONDS: i64 = 3_600;
pub(super) const JOB_RETRY_DELAY_SECONDS: i64 = 3_600;
pub(super) const THREAD_SCAN_LIMIT: usize = 5_000;
pub(super) const PRUNE_BATCH_SIZE: usize = 200;
/// Prompt used for phase 1 extraction.
pub(super) const PROMPT: &str = include_str!("../templates/memories/stage_one_system.md");
/// Fallback stage-1 rollout truncation limit (tokens) when model metadata
/// does not include a valid context window.
pub(super) const DEFAULT_ROLLOUT_TOKEN_LIMIT: usize = 150_000;
/// Portion of the model effective input window reserved for the stage-1
/// rollout input.
///
/// Keeping this below 100% leaves room for system instructions, prompt framing,
/// and model output.
pub(super) const CONTEXT_WINDOW_PERCENT: i64 = 70;
}
mod stage_two {
pub(super) const MODEL: &str = "gpt-5.4";
pub(super) const REASONING_EFFORT: codex_protocol::openai_models::ReasoningEffort =
codex_protocol::openai_models::ReasoningEffort::Medium;
pub(super) const JOB_LEASE_SECONDS: i64 = 3_600;
pub(super) const JOB_RETRY_DELAY_SECONDS: i64 = 3_600;
pub(super) const JOB_HEARTBEAT_SECONDS: u64 = 90;
}
mod workspace_diff {
/// Generated diff file the Phase 2 consolidation agent reads before editing memories.
pub(super) const FILENAME: &str = "phase2_workspace_diff.md";
pub(super) const MAX_BYTES: usize = 4 * 1024 * 1024;
}
pub fn memory_root(codex_home: &AbsolutePathBuf) -> AbsolutePathBuf {
codex_home.join("memories")
}
+8 -18
View File
@@ -1,4 +1,3 @@
use crate::STAGE_ONE_PROMPT;
use crate::build_stage_one_input_message;
use crate::metrics::MEMORY_PHASE_ONE_E2E_MS;
use crate::metrics::MEMORY_PHASE_ONE_JOBS;
@@ -14,7 +13,6 @@ use codex_protocol::error::CodexErr;
use codex_protocol::models::BaseInstructions;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::TokenUsage;
use codex_rollout::INTERACTIVE_SESSION_SOURCES;
@@ -29,14 +27,6 @@ use std::sync::Arc;
use tracing::info;
use tracing::warn;
const MODEL: &str = "gpt-5.4-mini";
const REASONING_EFFORT: ReasoningEffort = ReasoningEffort::Low;
const CONCURRENCY_LIMIT: usize = 8;
const JOB_LEASE_SECONDS: i64 = 3_600;
const JOB_RETRY_DELAY_SECONDS: i64 = 3_600;
const THREAD_SCAN_LIMIT: usize = 5_000;
const PRUNE_BATCH_SIZE: usize = 200;
struct JobResult {
outcome: JobOutcome,
token_usage: Option<TokenUsage>,
@@ -122,7 +112,7 @@ pub async fn prune(context: &MemoryStartupContext, config: &Config) {
if let Some(db) = context.state_db() {
let max_unused_days = config.memories.max_unused_days;
match db
.prune_stage1_outputs_for_retention(max_unused_days, PRUNE_BATCH_SIZE)
.prune_stage1_outputs_for_retention(max_unused_days, crate::stage_one::PRUNE_BATCH_SIZE)
.await
{
Ok(pruned) => {
@@ -174,12 +164,12 @@ async fn claim_startup_jobs(
.claim_stage1_jobs_for_startup(
context.thread_id(),
codex_state::Stage1StartupClaimParams {
scan_limit: THREAD_SCAN_LIMIT,
scan_limit: crate::stage_one::THREAD_SCAN_LIMIT,
max_claimed: memories_config.max_rollouts_per_startup,
max_age_days: memories_config.max_rollout_age_days,
min_rollout_idle_hours: memories_config.min_rollout_idle_hours,
allowed_sources: allowed_sources.as_slice(),
lease_seconds: JOB_LEASE_SECONDS,
lease_seconds: crate::stage_one::JOB_LEASE_SECONDS,
},
)
.await
@@ -200,9 +190,9 @@ async fn build_request_context(
.memories
.extract_model
.clone()
.unwrap_or(MODEL.to_string());
.unwrap_or(crate::stage_one::MODEL.to_string());
context
.stage_one_request_context(config, &model_name, REASONING_EFFORT)
.stage_one_request_context(config, &model_name, crate::stage_one::REASONING_EFFORT)
.await
}
@@ -221,7 +211,7 @@ async fn run_jobs(
job::run(context.as_ref(), config.as_ref(), claim, &stage_one_context).await
}
})
.buffer_unordered(CONCURRENCY_LIMIT)
.buffer_unordered(crate::stage_one::CONCURRENCY_LIMIT)
.collect::<Vec<_>>()
.await
}
@@ -310,7 +300,7 @@ mod job {
phase: None,
}];
prompt.base_instructions = BaseInstructions {
text: STAGE_ONE_PROMPT.to_string(),
text: crate::stage_one::PROMPT.to_string(),
};
prompt.output_schema = Some(output_schema());
prompt.output_schema_strict = true;
@@ -343,7 +333,7 @@ mod job {
thread_id,
ownership_token,
reason,
JOB_RETRY_DELAY_SECONDS,
crate::stage_one::JOB_RETRY_DELAY_SECONDS,
)
.await;
}
+16 -15
View File
@@ -28,12 +28,6 @@ use std::collections::HashMap;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
const MODEL: &str = "gpt-5.4";
const REASONING_EFFORT: codex_protocol::openai_models::ReasoningEffort =
codex_protocol::openai_models::ReasoningEffort::Medium;
const JOB_LEASE_SECONDS: i64 = 3_600;
const JOB_RETRY_DELAY_SECONDS: i64 = 3_600;
const JOB_HEARTBEAT_SECONDS: u64 = 90;
#[derive(Debug, Clone, Default)]
struct Claim {
@@ -223,7 +217,7 @@ mod job {
db: &StateRuntime,
) -> Result<Claim, &'static str> {
let claim = db
.try_claim_global_phase2_job(context.thread_id(), JOB_LEASE_SECONDS)
.try_claim_global_phase2_job(context.thread_id(), crate::stage_two::JOB_LEASE_SECONDS)
.await
.map_err(|e| {
tracing::error!("failed to claim job: {e}");
@@ -261,15 +255,19 @@ mod job {
) {
context.counter(MEMORY_PHASE_TWO_JOBS, /*inc*/ 1, &[("status", reason)]);
if matches!(
db.mark_global_phase2_job_failed(&claim.token, reason, JOB_RETRY_DELAY_SECONDS,)
.await,
db.mark_global_phase2_job_failed(
&claim.token,
reason,
crate::stage_two::JOB_RETRY_DELAY_SECONDS,
)
.await,
Ok(false)
) {
let _ = db
.mark_global_phase2_job_failed_if_unowned(
&claim.token,
reason,
JOB_RETRY_DELAY_SECONDS,
crate::stage_two::JOB_RETRY_DELAY_SECONDS,
)
.await;
}
@@ -336,9 +334,9 @@ mod agent {
.memories
.consolidation_model
.clone()
.unwrap_or(MODEL.to_string()),
.unwrap_or(crate::stage_two::MODEL.to_string()),
);
agent_config.model_reasoning_effort = Some(REASONING_EFFORT);
agent_config.model_reasoning_effort = Some(crate::stage_two::REASONING_EFFORT);
Some(agent_config)
}
@@ -384,7 +382,10 @@ mod agent {
}
// Do not reset the workspace baseline if we lost the lock.
let still_owns_lock = match db
.heartbeat_global_phase2_job(&claim.token, JOB_LEASE_SECONDS)
.heartbeat_global_phase2_job(
&claim.token,
crate::stage_two::JOB_LEASE_SECONDS,
)
.await
.inspect_err(|err| {
tracing::error!(
@@ -448,7 +449,7 @@ mod agent {
thread: &codex_core::CodexThread,
) -> AgentStatus {
let mut heartbeat_interval =
tokio::time::interval(Duration::from_secs(JOB_HEARTBEAT_SECONDS));
tokio::time::interval(Duration::from_secs(crate::stage_two::JOB_HEARTBEAT_SECONDS));
heartbeat_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut status_poll_interval = tokio::time::interval(Duration::from_secs(1));
status_poll_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
@@ -480,7 +481,7 @@ mod agent {
match db
.heartbeat_global_phase2_job(
&token,
JOB_LEASE_SECONDS,
crate::stage_two::JOB_LEASE_SECONDS,
)
.await
{
+5 -34
View File
@@ -1,7 +1,4 @@
use crate::DEFAULT_STAGE_ONE_ROLLOUT_TOKEN_LIMIT;
use crate::STAGE_ONE_CONTEXT_WINDOW_PERCENT;
use crate::memory_extensions_root;
use crate::workspace::WORKSPACE_DIFF_FILENAME;
use codex_protocol::openai_models::ModelInfo;
use codex_utils_output_truncation::TruncationPolicy;
use codex_utils_output_truncation::truncate_text;
@@ -24,13 +21,13 @@ static STAGE_ONE_INPUT_TEMPLATE: LazyLock<Template> = LazyLock::new(|| {
});
static MEMORY_EXTENSIONS_FOLDER_STRUCTURE_TEMPLATE: LazyLock<Template> = LazyLock::new(|| {
parse_embedded_template(
MEMORY_EXTENSIONS_FOLDER_STRUCTURE,
crate::prompt_blocks::EXTENSIONS_FOLDER_STRUCTURE,
"memories/extensions_folder_structure.md",
)
});
static MEMORY_EXTENSIONS_PRIMARY_INPUTS_TEMPLATE: LazyLock<Template> = LazyLock::new(|| {
parse_embedded_template(
MEMORY_EXTENSIONS_PRIMARY_INPUTS,
crate::prompt_blocks::EXTENSIONS_PRIMARY_INPUTS,
"memories/extensions_primary_inputs.md",
)
});
@@ -42,39 +39,13 @@ fn parse_embedded_template(source: &'static str, template_name: &str) -> Templat
}
}
const MEMORY_EXTENSIONS_FOLDER_STRUCTURE: &str = r#"
Memory extensions (under {{ memory_extensions_root }}/):
- <extension_name>/instructions.md
- Source-specific guidance for interpreting additional memory signals. If an
extension folder exists, you must read its instructions.md to determine how to use this memory
source.
If the user has any memory extensions, you MUST read the instructions for each extension to
determine how to use the memory source. If the workspace diff shows deleted extension resource files,
remove stale memories derived only from those resources. If it has no extension folders, continue
with the standard memory inputs only.
"#;
const MEMORY_EXTENSIONS_PRIMARY_INPUTS: &str = r#"
Optional source-specific inputs:
Under `{{ memory_extensions_root }}/`:
- `<extension_name>/instructions.md`
- If extension folders exist, read each instructions.md first and follow it when interpreting
that extension's memory source.
If the workspace diff shows deleted memory extension resources, use that extension-specific deletion
signal to remove stale memories derived only from those resources.
"#;
/// Builds the consolidation subagent prompt for a specific memory root.
pub fn build_consolidation_prompt(memory_root: &Path) -> String {
let memory_extensions_root = memory_extensions_root(memory_root);
let memory_extensions_exist = memory_extensions_root.is_dir();
let memory_root = memory_root.display().to_string();
let memory_extensions_root = memory_extensions_root.display().to_string();
let phase2_workspace_diff_file = WORKSPACE_DIFF_FILENAME.to_string();
let phase2_workspace_diff_file = crate::workspace_diff::FILENAME.to_string();
let memory_extensions_folder_structure = if memory_extensions_exist {
render_memory_extensions_block(
&MEMORY_EXTENSIONS_FOLDER_STRUCTURE_TEMPLATE,
@@ -138,9 +109,9 @@ pub fn build_stage_one_input_message(
.resolved_context_window()
.and_then(|limit| (limit > 0).then_some(limit))
.map(|limit| limit.saturating_mul(model_info.effective_context_window_percent) / 100)
.map(|limit| (limit.saturating_mul(STAGE_ONE_CONTEXT_WINDOW_PERCENT) / 100).max(1))
.map(|limit| (limit.saturating_mul(crate::stage_one::CONTEXT_WINDOW_PERCENT) / 100).max(1))
.and_then(|limit| usize::try_from(limit).ok())
.unwrap_or(DEFAULT_STAGE_ONE_ROLLOUT_TOKEN_LIMIT);
.unwrap_or(crate::stage_one::DEFAULT_ROLLOUT_TOKEN_LIMIT);
let truncated_rollout_contents = truncate_text(
rollout_contents,
TruncationPolicy::Tokens(rollout_token_limit),
+2 -2
View File
@@ -9,7 +9,7 @@ fn build_stage_one_input_message_truncates_rollout_using_model_context_window()
model_info.context_window = Some(123_000);
let expected_rollout_token_limit = usize::try_from(
((123_000_i64 * model_info.effective_context_window_percent) / 100)
* STAGE_ONE_CONTEXT_WINDOW_PERCENT
* crate::stage_one::CONTEXT_WINDOW_PERCENT
/ 100,
)
.unwrap();
@@ -39,7 +39,7 @@ fn build_stage_one_input_message_uses_default_limit_when_model_context_window_mi
model_info.max_context_window = None;
let expected_truncated = truncate_text(
&input,
TruncationPolicy::Tokens(DEFAULT_STAGE_ONE_ROLLOUT_TOKEN_LIMIT),
TruncationPolicy::Tokens(crate::stage_one::DEFAULT_ROLLOUT_TOKEN_LIMIT),
);
let message = build_stage_one_input_message(
&model_info,
+6 -10
View File
@@ -5,11 +5,6 @@ use codex_git_utils::ensure_git_baseline_repository;
use codex_git_utils::reset_git_repository;
use std::path::Path;
/// Generated diff file the Phase 2 consolidation agent reads before editing memories.
pub const WORKSPACE_DIFF_FILENAME: &str = "phase2_workspace_diff.md";
const WORKSPACE_DIFF_MAX_BYTES: usize = 4 * 1024 * 1024;
/// Prepares the memory directory for git-baseline diffing.
///
/// This keeps an existing usable `.git/` baseline intact. It initializes a new git baseline when the
@@ -35,7 +30,7 @@ pub async fn memory_workspace_diff(root: &Path) -> anyhow::Result<GitBaselineDif
/// Writes `phase2_workspace_diff.md` with a bounded git-style diff from the current baseline.
pub async fn write_workspace_diff(root: &Path, diff: &GitBaselineDiff) -> anyhow::Result<()> {
let path = root.join(WORKSPACE_DIFF_FILENAME);
let path = root.join(crate::workspace_diff::FILENAME);
tokio::fs::write(&path, render_workspace_diff_file(diff))
.await
.with_context(|| format!("write memory workspace diff file {}", path.display()))
@@ -56,7 +51,7 @@ pub async fn reset_memory_workspace_baseline(root: &Path) -> anyhow::Result<()>
/// diffing and before baseline reset so the generated diff file itself is not treated as memory
/// workspace input.
pub(super) async fn remove_workspace_diff(root: &Path) -> anyhow::Result<()> {
let path = root.join(WORKSPACE_DIFF_FILENAME);
let path = root.join(crate::workspace_diff::FILENAME);
match tokio::fs::remove_file(&path).await {
Ok(()) => Ok(()),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
@@ -87,7 +82,7 @@ fn render_workspace_diff_file(diff: &GitBaselineDiff) -> String {
}
fn append_bounded_diff(rendered: &mut String, diff: &str) {
if diff.len() <= WORKSPACE_DIFF_MAX_BYTES {
if diff.len() <= crate::workspace_diff::MAX_BYTES {
rendered.push_str(diff);
if !diff.ends_with('\n') {
rendered.push('\n');
@@ -95,13 +90,14 @@ fn append_bounded_diff(rendered: &mut String, diff: &str) {
return;
}
let boundary = previous_char_boundary(diff, WORKSPACE_DIFF_MAX_BYTES);
let boundary = previous_char_boundary(diff, crate::workspace_diff::MAX_BYTES);
rendered.push_str(&diff[..boundary]);
if !rendered.ends_with('\n') {
rendered.push('\n');
}
rendered.push_str(&format!(
"\n[workspace diff truncated at {WORKSPACE_DIFF_MAX_BYTES} bytes]\n"
"\n[workspace diff truncated at {} bytes]\n",
crate::workspace_diff::MAX_BYTES
));
}
@@ -12,7 +12,7 @@ fn render_workspace_diff_file_bounds_large_diff() {
status: GitBaselineChangeStatus::Modified,
path: "MEMORY.md".to_string(),
}],
unified_diff: "a".repeat(WORKSPACE_DIFF_MAX_BYTES + 128),
unified_diff: "a".repeat(crate::workspace_diff::MAX_BYTES + 128),
};
let rendered = render_workspace_diff_file(&diff);
@@ -47,7 +47,7 @@ async fn reset_memory_workspace_baseline_removes_generated_diff() {
.await
.expect("reset baseline");
assert!(!root.join(WORKSPACE_DIFF_FILENAME).exists());
assert!(!root.join(crate::workspace_diff::FILENAME).exists());
let diff = memory_workspace_diff(&root)
.await
.expect("load workspace diff");