From 5b7d6f5c4f5520d1abbf04df393ef18f0914c0f5 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 28 Apr 2026 18:13:35 +0200 Subject: [PATCH] feat: house-keeping memories 3 (#20005) Move stuff in memories, no behavioural change expected --- codex-rs/memories/write/src/extensions.rs | 9 +- .../memories/write/src/extensions_tests.rs | 7 +- codex-rs/memories/write/src/guard.rs | 4 +- codex-rs/memories/write/src/guard_tests.rs | 2 +- codex-rs/memories/write/src/lib.rs | 91 ++++++++++++++++--- codex-rs/memories/write/src/phase1.rs | 26 ++---- codex-rs/memories/write/src/phase2.rs | 31 ++++--- codex-rs/memories/write/src/prompts.rs | 39 +------- codex-rs/memories/write/src/prompts_tests.rs | 4 +- codex-rs/memories/write/src/workspace.rs | 16 ++-- .../memories/write/src/workspace_tests.rs | 4 +- 11 files changed, 127 insertions(+), 106 deletions(-) diff --git a/codex-rs/memories/write/src/extensions.rs b/codex-rs/memories/write/src/extensions.rs index f764b19c1..7b770cdf0 100644 --- a/codex-rs/memories/write/src/extensions.rs +++ b/codex-rs/memories/write/src/extensions.rs @@ -6,15 +6,12 @@ use chrono::Utc; use std::path::Path; use tracing::warn; -const FILENAME_TS_FORMAT: &str = "%Y-%m-%dT%H-%M-%S"; -const EXTENSION_RESOURCE_RETENTION_DAYS: i64 = 7; - pub async fn prune_old_extension_resources(memory_root: &Path) { prune_old_extension_resources_with_now(memory_root, Utc::now()).await } async fn prune_old_extension_resources_with_now(memory_root: &Path, now: DateTime) { - let cutoff = now - Duration::days(EXTENSION_RESOURCE_RETENTION_DAYS); + let cutoff = now - Duration::days(crate::extension_resources::RETENTION_DAYS); let extensions_root = memory_extensions_root(memory_root); let mut extensions = match tokio::fs::read_dir(&extensions_root).await { Ok(extensions) => extensions, @@ -92,7 +89,9 @@ async fn prune_old_extension_resources_with_now(memory_root: &Path, now: DateTim fn resource_timestamp(file_name: &str) -> Option> { let timestamp = file_name.get(..19)?; - let naive = NaiveDateTime::parse_from_str(timestamp, FILENAME_TS_FORMAT).ok()?; + let naive = + NaiveDateTime::parse_from_str(timestamp, crate::extension_resources::FILENAME_TS_FORMAT) + .ok()?; Some(DateTime::from_naive_utc_and_offset(naive, Utc)) } diff --git a/codex-rs/memories/write/src/extensions_tests.rs b/codex-rs/memories/write/src/extensions_tests.rs index 60cd18757..e93335e16 100644 --- a/codex-rs/memories/write/src/extensions_tests.rs +++ b/codex-rs/memories/write/src/extensions_tests.rs @@ -19,8 +19,11 @@ async fn prunes_only_old_resources_from_extensions_with_instructions() { .expect("write chronicle instructions"); let now = DateTime::from_naive_utc_and_offset( - NaiveDateTime::parse_from_str("2026-04-14T12-00-00", FILENAME_TS_FORMAT) - .expect("parse now"), + NaiveDateTime::parse_from_str( + "2026-04-14T12-00-00", + crate::extension_resources::FILENAME_TS_FORMAT, + ) + .expect("parse now"), Utc, ); let old_file = chronicle_resources.join("2026-04-06T11-59-59-abcd-10min-old.md"); diff --git a/codex-rs/memories/write/src/guard.rs b/codex-rs/memories/write/src/guard.rs index 7deb74517..4d75043f9 100644 --- a/codex-rs/memories/write/src/guard.rs +++ b/codex-rs/memories/write/src/guard.rs @@ -6,8 +6,6 @@ use codex_protocol::protocol::RateLimitWindow; use tracing::info; use tracing::warn; -const CODEX_LIMIT_ID: &str = "codex"; - pub(crate) async fn rate_limits_ok(auth_manager: &AuthManager, config: &Config) -> bool { rate_limits_check(auth_manager, config) .await @@ -32,7 +30,7 @@ async fn rate_limits_check(auth_manager: &AuthManager, config: &Config) -> Optio let snapshot = snapshots .iter() - .find(|s| s.limit_id.as_deref() == Some(CODEX_LIMIT_ID)) + .find(|s| s.limit_id.as_deref() == Some(crate::guard_limits::CODEX_LIMIT_ID)) .or_else(|| snapshots.first())?; let min_remaining_percent = config.memories.min_rate_limit_remaining_percent; diff --git a/codex-rs/memories/write/src/guard_tests.rs b/codex-rs/memories/write/src/guard_tests.rs index 6c22b0681..139a8773c 100644 --- a/codex-rs/memories/write/src/guard_tests.rs +++ b/codex-rs/memories/write/src/guard_tests.rs @@ -6,7 +6,7 @@ fn snapshot( secondary_used_percent: Option, ) -> RateLimitSnapshot { RateLimitSnapshot { - limit_id: Some(CODEX_LIMIT_ID.to_string()), + limit_id: Some(crate::guard_limits::CODEX_LIMIT_ID.to_string()), limit_name: None, primary: primary_used_percent.map(window), secondary: secondary_used_percent.map(window), diff --git a/codex-rs/memories/write/src/lib.rs b/codex-rs/memories/write/src/lib.rs index 63f156571..6764ebf5c 100644 --- a/codex-rs/memories/write/src/lib.rs +++ b/codex-rs/memories/write/src/lib.rs @@ -29,20 +29,6 @@ pub use storage::rebuild_raw_memories_file_from_memories; pub use storage::rollout_summary_file_stem; pub use storage::sync_rollout_summaries_from_memories; -/// Prompt used for phase 1 extraction. -pub const STAGE_ONE_PROMPT: &str = include_str!("../templates/memories/stage_one_system.md"); - -/// Fallback stage-1 rollout truncation limit (tokens) when model metadata -/// does not include a valid context window. -pub const DEFAULT_STAGE_ONE_ROLLOUT_TOKEN_LIMIT: usize = 150_000; - -/// Portion of the model effective input window reserved for the stage-1 -/// rollout input. -/// -/// Keeping this below 100% leaves room for system instructions, prompt framing, -/// and model output. -pub const STAGE_ONE_CONTEXT_WINDOW_PERCENT: i64 = 70; - #[cfg(test)] mod startup_tests; @@ -52,6 +38,83 @@ mod artifacts { pub(super) const RAW_MEMORIES_FILENAME: &str = "raw_memories.md"; } +mod extension_resources { + pub(super) const FILENAME_TS_FORMAT: &str = "%Y-%m-%dT%H-%M-%S"; + pub(super) const RETENTION_DAYS: i64 = 7; +} + +mod guard_limits { + pub(super) const CODEX_LIMIT_ID: &str = "codex"; +} + +mod prompt_blocks { + pub(super) const EXTENSIONS_FOLDER_STRUCTURE: &str = r#" +Memory extensions (under {{ memory_extensions_root }}/): + +- /instructions.md + - Source-specific guidance for interpreting additional memory signals. If an + extension folder exists, you must read its instructions.md to determine how to use this memory + source. + +If the user has any memory extensions, you MUST read the instructions for each extension to +determine how to use the memory source. If the workspace diff shows deleted extension resource files, +remove stale memories derived only from those resources. If it has no extension folders, continue +with the standard memory inputs only. +"#; + + pub(super) const EXTENSIONS_PRIMARY_INPUTS: &str = r#" +Optional source-specific inputs: +Under `{{ memory_extensions_root }}/`: + +- `/instructions.md` + - If extension folders exist, read each instructions.md first and follow it when interpreting + that extension's memory source. + +If the workspace diff shows deleted memory extension resources, use that extension-specific deletion +signal to remove stale memories derived only from those resources. +"#; +} + +mod stage_one { + pub(super) const MODEL: &str = "gpt-5.4-mini"; + pub(super) const REASONING_EFFORT: codex_protocol::openai_models::ReasoningEffort = + codex_protocol::openai_models::ReasoningEffort::Low; + pub(super) const CONCURRENCY_LIMIT: usize = 8; + pub(super) const JOB_LEASE_SECONDS: i64 = 3_600; + pub(super) const JOB_RETRY_DELAY_SECONDS: i64 = 3_600; + pub(super) const THREAD_SCAN_LIMIT: usize = 5_000; + pub(super) const PRUNE_BATCH_SIZE: usize = 200; + + /// Prompt used for phase 1 extraction. + pub(super) const PROMPT: &str = include_str!("../templates/memories/stage_one_system.md"); + + /// Fallback stage-1 rollout truncation limit (tokens) when model metadata + /// does not include a valid context window. + pub(super) const DEFAULT_ROLLOUT_TOKEN_LIMIT: usize = 150_000; + + /// Portion of the model effective input window reserved for the stage-1 + /// rollout input. + /// + /// Keeping this below 100% leaves room for system instructions, prompt framing, + /// and model output. + pub(super) const CONTEXT_WINDOW_PERCENT: i64 = 70; +} + +mod stage_two { + pub(super) const MODEL: &str = "gpt-5.4"; + pub(super) const REASONING_EFFORT: codex_protocol::openai_models::ReasoningEffort = + codex_protocol::openai_models::ReasoningEffort::Medium; + pub(super) const JOB_LEASE_SECONDS: i64 = 3_600; + pub(super) const JOB_RETRY_DELAY_SECONDS: i64 = 3_600; + pub(super) const JOB_HEARTBEAT_SECONDS: u64 = 90; +} + +mod workspace_diff { + /// Generated diff file the Phase 2 consolidation agent reads before editing memories. + pub(super) const FILENAME: &str = "phase2_workspace_diff.md"; + pub(super) const MAX_BYTES: usize = 4 * 1024 * 1024; +} + pub fn memory_root(codex_home: &AbsolutePathBuf) -> AbsolutePathBuf { codex_home.join("memories") } diff --git a/codex-rs/memories/write/src/phase1.rs b/codex-rs/memories/write/src/phase1.rs index a3a85b8e8..9c6c2561d 100644 --- a/codex-rs/memories/write/src/phase1.rs +++ b/codex-rs/memories/write/src/phase1.rs @@ -1,4 +1,3 @@ -use crate::STAGE_ONE_PROMPT; use crate::build_stage_one_input_message; use crate::metrics::MEMORY_PHASE_ONE_E2E_MS; use crate::metrics::MEMORY_PHASE_ONE_JOBS; @@ -14,7 +13,6 @@ use codex_protocol::error::CodexErr; use codex_protocol::models::BaseInstructions; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; -use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::TokenUsage; use codex_rollout::INTERACTIVE_SESSION_SOURCES; @@ -29,14 +27,6 @@ use std::sync::Arc; use tracing::info; use tracing::warn; -const MODEL: &str = "gpt-5.4-mini"; -const REASONING_EFFORT: ReasoningEffort = ReasoningEffort::Low; -const CONCURRENCY_LIMIT: usize = 8; -const JOB_LEASE_SECONDS: i64 = 3_600; -const JOB_RETRY_DELAY_SECONDS: i64 = 3_600; -const THREAD_SCAN_LIMIT: usize = 5_000; -const PRUNE_BATCH_SIZE: usize = 200; - struct JobResult { outcome: JobOutcome, token_usage: Option, @@ -122,7 +112,7 @@ pub async fn prune(context: &MemoryStartupContext, config: &Config) { if let Some(db) = context.state_db() { let max_unused_days = config.memories.max_unused_days; match db - .prune_stage1_outputs_for_retention(max_unused_days, PRUNE_BATCH_SIZE) + .prune_stage1_outputs_for_retention(max_unused_days, crate::stage_one::PRUNE_BATCH_SIZE) .await { Ok(pruned) => { @@ -174,12 +164,12 @@ async fn claim_startup_jobs( .claim_stage1_jobs_for_startup( context.thread_id(), codex_state::Stage1StartupClaimParams { - scan_limit: THREAD_SCAN_LIMIT, + scan_limit: crate::stage_one::THREAD_SCAN_LIMIT, max_claimed: memories_config.max_rollouts_per_startup, max_age_days: memories_config.max_rollout_age_days, min_rollout_idle_hours: memories_config.min_rollout_idle_hours, allowed_sources: allowed_sources.as_slice(), - lease_seconds: JOB_LEASE_SECONDS, + lease_seconds: crate::stage_one::JOB_LEASE_SECONDS, }, ) .await @@ -200,9 +190,9 @@ async fn build_request_context( .memories .extract_model .clone() - .unwrap_or(MODEL.to_string()); + .unwrap_or(crate::stage_one::MODEL.to_string()); context - .stage_one_request_context(config, &model_name, REASONING_EFFORT) + .stage_one_request_context(config, &model_name, crate::stage_one::REASONING_EFFORT) .await } @@ -221,7 +211,7 @@ async fn run_jobs( job::run(context.as_ref(), config.as_ref(), claim, &stage_one_context).await } }) - .buffer_unordered(CONCURRENCY_LIMIT) + .buffer_unordered(crate::stage_one::CONCURRENCY_LIMIT) .collect::>() .await } @@ -310,7 +300,7 @@ mod job { phase: None, }]; prompt.base_instructions = BaseInstructions { - text: STAGE_ONE_PROMPT.to_string(), + text: crate::stage_one::PROMPT.to_string(), }; prompt.output_schema = Some(output_schema()); prompt.output_schema_strict = true; @@ -343,7 +333,7 @@ mod job { thread_id, ownership_token, reason, - JOB_RETRY_DELAY_SECONDS, + crate::stage_one::JOB_RETRY_DELAY_SECONDS, ) .await; } diff --git a/codex-rs/memories/write/src/phase2.rs b/codex-rs/memories/write/src/phase2.rs index 14954c49e..7a3841f8c 100644 --- a/codex-rs/memories/write/src/phase2.rs +++ b/codex-rs/memories/write/src/phase2.rs @@ -28,12 +28,6 @@ use std::collections::HashMap; use std::path::Path; use std::sync::Arc; use std::time::Duration; -const MODEL: &str = "gpt-5.4"; -const REASONING_EFFORT: codex_protocol::openai_models::ReasoningEffort = - codex_protocol::openai_models::ReasoningEffort::Medium; -const JOB_LEASE_SECONDS: i64 = 3_600; -const JOB_RETRY_DELAY_SECONDS: i64 = 3_600; -const JOB_HEARTBEAT_SECONDS: u64 = 90; #[derive(Debug, Clone, Default)] struct Claim { @@ -223,7 +217,7 @@ mod job { db: &StateRuntime, ) -> Result { let claim = db - .try_claim_global_phase2_job(context.thread_id(), JOB_LEASE_SECONDS) + .try_claim_global_phase2_job(context.thread_id(), crate::stage_two::JOB_LEASE_SECONDS) .await .map_err(|e| { tracing::error!("failed to claim job: {e}"); @@ -261,15 +255,19 @@ mod job { ) { context.counter(MEMORY_PHASE_TWO_JOBS, /*inc*/ 1, &[("status", reason)]); if matches!( - db.mark_global_phase2_job_failed(&claim.token, reason, JOB_RETRY_DELAY_SECONDS,) - .await, + db.mark_global_phase2_job_failed( + &claim.token, + reason, + crate::stage_two::JOB_RETRY_DELAY_SECONDS, + ) + .await, Ok(false) ) { let _ = db .mark_global_phase2_job_failed_if_unowned( &claim.token, reason, - JOB_RETRY_DELAY_SECONDS, + crate::stage_two::JOB_RETRY_DELAY_SECONDS, ) .await; } @@ -336,9 +334,9 @@ mod agent { .memories .consolidation_model .clone() - .unwrap_or(MODEL.to_string()), + .unwrap_or(crate::stage_two::MODEL.to_string()), ); - agent_config.model_reasoning_effort = Some(REASONING_EFFORT); + agent_config.model_reasoning_effort = Some(crate::stage_two::REASONING_EFFORT); Some(agent_config) } @@ -384,7 +382,10 @@ mod agent { } // Do not reset the workspace baseline if we lost the lock. let still_owns_lock = match db - .heartbeat_global_phase2_job(&claim.token, JOB_LEASE_SECONDS) + .heartbeat_global_phase2_job( + &claim.token, + crate::stage_two::JOB_LEASE_SECONDS, + ) .await .inspect_err(|err| { tracing::error!( @@ -448,7 +449,7 @@ mod agent { thread: &codex_core::CodexThread, ) -> AgentStatus { let mut heartbeat_interval = - tokio::time::interval(Duration::from_secs(JOB_HEARTBEAT_SECONDS)); + tokio::time::interval(Duration::from_secs(crate::stage_two::JOB_HEARTBEAT_SECONDS)); heartbeat_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let mut status_poll_interval = tokio::time::interval(Duration::from_secs(1)); status_poll_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -480,7 +481,7 @@ mod agent { match db .heartbeat_global_phase2_job( &token, - JOB_LEASE_SECONDS, + crate::stage_two::JOB_LEASE_SECONDS, ) .await { diff --git a/codex-rs/memories/write/src/prompts.rs b/codex-rs/memories/write/src/prompts.rs index 4f607f821..e0019f610 100644 --- a/codex-rs/memories/write/src/prompts.rs +++ b/codex-rs/memories/write/src/prompts.rs @@ -1,7 +1,4 @@ -use crate::DEFAULT_STAGE_ONE_ROLLOUT_TOKEN_LIMIT; -use crate::STAGE_ONE_CONTEXT_WINDOW_PERCENT; use crate::memory_extensions_root; -use crate::workspace::WORKSPACE_DIFF_FILENAME; use codex_protocol::openai_models::ModelInfo; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::truncate_text; @@ -24,13 +21,13 @@ static STAGE_ONE_INPUT_TEMPLATE: LazyLock