mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
chore: split memories part 1 (#19818)
Extract memories into 2 different crates
This commit is contained in:
jif-oai
committed by
GitHub
Unverified
GitHub
parent
f431ec12c9
commit
bb83eec825
@@ -1,7 +1,6 @@
|
||||
use crate::agents_md::AgentsMdManager;
|
||||
use crate::config::edit::ConfigEdit;
|
||||
use crate::config::edit::ConfigEditsBuilder;
|
||||
use crate::memories::memory_root;
|
||||
use crate::path_utils::normalize_for_native_workdir;
|
||||
use crate::unified_exec::DEFAULT_MAX_BACKGROUND_TERMINAL_TIMEOUT_MS;
|
||||
use crate::unified_exec::MIN_EMPTY_YIELD_TIME_MS;
|
||||
@@ -63,6 +62,7 @@ use codex_features::MultiAgentV2ConfigToml;
|
||||
use codex_git_utils::resolve_root_git_project_for_trust;
|
||||
use codex_login::AuthManagerConfig;
|
||||
use codex_mcp::McpConfig;
|
||||
use codex_memories_write::memory_root;
|
||||
use codex_model_provider_info::LEGACY_OLLAMA_CHAT_PROVIDER_ID;
|
||||
use codex_model_provider_info::ModelProviderInfo;
|
||||
use codex_model_provider_info::OLLAMA_CHAT_PROVIDER_REMOVED_ERROR;
|
||||
|
||||
@@ -57,7 +57,7 @@ pub use codex_mcp::SandboxState;
|
||||
mod mcp_openai_file;
|
||||
mod mcp_tool_call;
|
||||
mod memories;
|
||||
pub use memories::clear_memory_roots_contents;
|
||||
pub use codex_memories_write::clear_memory_roots_contents;
|
||||
pub(crate) mod mention_syntax;
|
||||
pub(crate) mod message_history;
|
||||
pub(crate) mod utils;
|
||||
@@ -200,4 +200,5 @@ pub mod compact;
|
||||
pub(crate) mod memory_trace;
|
||||
pub use memory_trace::BuiltMemory;
|
||||
pub use memory_trace::build_memories_from_trace_files;
|
||||
mod memory_usage;
|
||||
pub mod otel_init;
|
||||
|
||||
@@ -1,144 +0,0 @@
|
||||
# Memories Pipeline (Core)
|
||||
|
||||
This module runs a startup memory pipeline for eligible sessions.
|
||||
|
||||
## Prompt Templates
|
||||
|
||||
Memory prompt templates live under `codex-rs/core/templates/memories/`.
|
||||
|
||||
- The undated template files are the canonical latest versions used at runtime:
|
||||
- `stage_one_system.md`
|
||||
- `stage_one_input.md`
|
||||
- `consolidation.md`
|
||||
- `read_path.md`
|
||||
- In `codex`, edit those undated template files in place.
|
||||
- The dated snapshot-copy workflow is used in the separate `openai/project/agent_memory/write` harness repo, not here.
|
||||
|
||||
## When it runs
|
||||
|
||||
The pipeline is triggered when a root session starts, and only if:
|
||||
|
||||
- the session is not ephemeral
|
||||
- the memory feature is enabled
|
||||
- the session is not a sub-agent session
|
||||
- the state DB is available
|
||||
|
||||
It runs asynchronously in the background and executes two phases in order: Phase 1, then Phase 2.
|
||||
|
||||
## Phase 1: Rollout Extraction (per-thread)
|
||||
|
||||
Phase 1 finds recent eligible rollouts and extracts a structured memory from each one.
|
||||
|
||||
Eligible rollouts are selected from the state DB using startup claim rules. In practice this means
|
||||
the pipeline only considers rollouts that are:
|
||||
|
||||
- from allowed interactive session sources
|
||||
- within the configured age window
|
||||
- idle long enough (to avoid summarizing still-active/fresh rollouts)
|
||||
- not already owned by another in-flight phase-1 worker
|
||||
- within startup scan/claim limits (bounded work per startup)
|
||||
|
||||
What it does:
|
||||
|
||||
- claims a bounded set of rollout jobs from the state DB (startup claim)
|
||||
- filters rollout content down to memory-relevant response items
|
||||
- sends each rollout to a model (in parallel, with a concurrency cap)
|
||||
- expects structured output containing:
|
||||
- a detailed `raw_memory`
|
||||
- a compact `rollout_summary`
|
||||
- an optional `rollout_slug`
|
||||
- redacts secrets from the generated memory fields
|
||||
- stores successful outputs back into the state DB as stage-1 outputs
|
||||
|
||||
Concurrency / coordination:
|
||||
|
||||
- Phase 1 runs multiple extraction jobs in parallel (with a fixed concurrency cap) so startup memory generation can process several rollouts at once.
|
||||
- Each job is leased/claimed in the state DB before processing, which prevents duplicate work across concurrent workers/startups.
|
||||
- Failed jobs are marked with retry backoff, so they are retried later instead of hot-looping.
|
||||
|
||||
Job outcomes:
|
||||
|
||||
- `succeeded` (memory produced)
|
||||
- `succeeded_no_output` (valid run but nothing useful generated)
|
||||
- `failed` (with retry backoff/lease handling in DB)
|
||||
|
||||
Phase 1 is the stage that turns individual rollouts into DB-backed memory records.
|
||||
|
||||
## Phase 2: Global Consolidation
|
||||
|
||||
Phase 2 consolidates the latest stage-1 outputs into the filesystem memory artifacts and then runs a dedicated consolidation agent.
|
||||
|
||||
What it does:
|
||||
|
||||
- claims a single global phase-2 lock before touching the memories root (so only one consolidation
|
||||
inspects or mutates the workspace at a time)
|
||||
- loads a bounded set of stage-1 outputs from the state DB using phase-2
|
||||
selection rules:
|
||||
- ignores memories whose `last_usage` falls outside the configured
|
||||
`max_unused_days` window
|
||||
- for memories with no `last_usage`, falls back to `generated_at` so fresh
|
||||
never-used memories can still be selected
|
||||
- ranks eligible memories by `usage_count` first, then by the most recent
|
||||
`last_usage` / `generated_at`
|
||||
- computes a completion watermark from the claimed watermark + newest input timestamps
|
||||
- syncs local memory artifacts under the memories root:
|
||||
- `raw_memories.md` (merged raw memories, latest first)
|
||||
- `rollout_summaries/` (one summary file per selected rollout)
|
||||
- keeps the memories root itself as a git-baseline directory, initialized under
|
||||
`~/.codex/memories/.git` by `codex-git-utils`
|
||||
- prunes stale rollout summaries that are no longer selected
|
||||
- prunes memory extension resource files older than the extension retention
|
||||
window, so cleanup appears in the workspace diff
|
||||
- writes `phase2_workspace_diff.md` in the memories root with the git-style diff
|
||||
from the previous successful Phase 2 baseline to the current worktree
|
||||
- if the memory workspace has no changes after artifact sync/pruning, marks the
|
||||
job successful and exits
|
||||
|
||||
If the memory workspace has changes, it then:
|
||||
|
||||
- spawns an internal consolidation sub-agent
|
||||
- builds the Phase 2 prompt with the path to the generated workspace diff
|
||||
- points the agent at `phase2_workspace_diff.md` for the detailed diff context
|
||||
- runs it with no approvals, no network, and local write access only
|
||||
- disables collab for that agent (to prevent recursive delegation)
|
||||
- watches the agent status and heartbeats the global job lease while it runs
|
||||
- resets the memory git baseline after the agent completes successfully; the
|
||||
generated diff file is removed before this reset so deleted content is not
|
||||
kept in the prompt artifact or unreachable git objects
|
||||
- marks the phase-2 job success/failure in the state DB when the agent finishes
|
||||
|
||||
Selection and workspace-diff behavior:
|
||||
|
||||
- successful Phase 2 runs mark the exact stage-1 snapshots they consumed with
|
||||
`selected_for_phase2 = 1` and persist the matching
|
||||
`selected_for_phase2_source_updated_at`
|
||||
- Phase 1 upserts preserve the previous `selected_for_phase2` baseline until
|
||||
the next successful Phase 2 run rewrites it
|
||||
- Phase 2 loads only the current top-N selected stage-1 inputs, syncs
|
||||
`rollout_summaries/` and `raw_memories.md` directly to that selection, then
|
||||
lets the git-style workspace diff surface additions, modifications, and
|
||||
deletions against the previous successful memory baseline
|
||||
- when the selected input set is empty, stale `rollout_summaries/` files are
|
||||
removed and `raw_memories.md` is rewritten to the empty-input placeholder;
|
||||
consolidated outputs such as `MEMORY.md`, `memory_summary.md`, and `skills/`
|
||||
are left for the agent to update
|
||||
|
||||
Watermark behavior:
|
||||
|
||||
- The global phase-2 lock does not use DB watermarks as a dirty check; git
|
||||
workspace dirtiness decides whether an agent needs to run.
|
||||
- The global phase-2 job row still tracks an input watermark as bookkeeping
|
||||
for the latest DB input timestamp known when the job was claimed.
|
||||
- Phase 2 recomputes a `new_watermark` using the max of:
|
||||
- the claimed watermark
|
||||
- the newest `source_updated_at` timestamp in the stage-1 inputs it actually loaded
|
||||
- On success, Phase 2 stores that completion watermark in the DB.
|
||||
- This avoids moving the recorded completion watermark backwards, but does not
|
||||
decide whether Phase 2 has work.
|
||||
|
||||
In practice, this phase is responsible for refreshing the on-disk memory workspace and producing/updating the higher-level consolidated memory outputs.
|
||||
|
||||
## Why it is split into two phases
|
||||
|
||||
- Phase 1 scales across many rollouts and produces normalized per-rollout memory records.
|
||||
- Phase 2 serializes global consolidation so the shared memory artifacts are updated safely and consistently.
|
||||
@@ -1,85 +0,0 @@
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::memory_citation::MemoryCitation;
|
||||
use codex_protocol::memory_citation::MemoryCitationEntry;
|
||||
use std::collections::HashSet;
|
||||
|
||||
pub fn parse_memory_citation(citations: Vec<String>) -> Option<MemoryCitation> {
|
||||
let mut entries = Vec::new();
|
||||
let mut rollout_ids = Vec::new();
|
||||
let mut seen_rollout_ids = HashSet::new();
|
||||
|
||||
for citation in citations {
|
||||
if let Some(entries_block) =
|
||||
extract_block(&citation, "<citation_entries>", "</citation_entries>")
|
||||
{
|
||||
entries.extend(
|
||||
entries_block
|
||||
.lines()
|
||||
.filter_map(parse_memory_citation_entry),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(ids_block) = extract_ids_block(&citation) {
|
||||
for id in ids_block
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| !line.is_empty())
|
||||
{
|
||||
if seen_rollout_ids.insert(id.to_string()) {
|
||||
rollout_ids.push(id.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if entries.is_empty() && rollout_ids.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(MemoryCitation {
|
||||
entries,
|
||||
rollout_ids,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub fn thread_ids_from_memory_citation(memory_citation: &MemoryCitation) -> Vec<ThreadId> {
|
||||
memory_citation
|
||||
.rollout_ids
|
||||
.iter()
|
||||
.filter_map(|id| ThreadId::try_from(id.as_str()).ok())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse_memory_citation_entry(line: &str) -> Option<MemoryCitationEntry> {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let (location, note) = line.rsplit_once("|note=[")?;
|
||||
let note = note.strip_suffix(']')?.trim().to_string();
|
||||
let (path, line_range) = location.rsplit_once(':')?;
|
||||
let (line_start, line_end) = line_range.split_once('-')?;
|
||||
|
||||
Some(MemoryCitationEntry {
|
||||
path: path.trim().to_string(),
|
||||
line_start: line_start.trim().parse().ok()?,
|
||||
line_end: line_end.trim().parse().ok()?,
|
||||
note,
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_block<'a>(text: &'a str, open: &str, close: &str) -> Option<&'a str> {
|
||||
let (_, rest) = text.split_once(open)?;
|
||||
let (body, _) = rest.split_once(close)?;
|
||||
Some(body)
|
||||
}
|
||||
|
||||
fn extract_ids_block(text: &str) -> Option<&str> {
|
||||
extract_block(text, "<rollout_ids>", "</rollout_ids>")
|
||||
.or_else(|| extract_block(text, "<thread_ids>", "</thread_ids>"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "citations_tests.rs"]
|
||||
mod tests;
|
||||
@@ -1,71 +0,0 @@
|
||||
use super::parse_memory_citation;
|
||||
use super::thread_ids_from_memory_citation;
|
||||
use codex_protocol::ThreadId;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn parse_memory_citation_supports_legacy_thread_ids() {
|
||||
let first = ThreadId::new();
|
||||
let second = ThreadId::new();
|
||||
|
||||
let citations = vec![format!(
|
||||
"<memory_citation>\n<citation_entries>\nMEMORY.md:1-2|note=[x]\n</citation_entries>\n<thread_ids>\n{first}\nnot-a-uuid\n{second}\n</thread_ids>\n</memory_citation>"
|
||||
)];
|
||||
|
||||
let parsed = parse_memory_citation(citations).expect("memory citation should parse");
|
||||
|
||||
assert_eq!(
|
||||
thread_ids_from_memory_citation(&parsed),
|
||||
vec![first, second]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_memory_citation_supports_rollout_ids() {
|
||||
let thread_id = ThreadId::new();
|
||||
|
||||
let citations = vec![format!(
|
||||
"<memory_citation>\n<rollout_ids>\n{thread_id}\n</rollout_ids>\n</memory_citation>"
|
||||
)];
|
||||
|
||||
let parsed = parse_memory_citation(citations).expect("memory citation should parse");
|
||||
|
||||
assert_eq!(thread_ids_from_memory_citation(&parsed), vec![thread_id]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_memory_citation_extracts_entries_and_rollout_ids() {
|
||||
let first = ThreadId::new();
|
||||
let second = ThreadId::new();
|
||||
let citations = vec![format!(
|
||||
"<citation_entries>\nMEMORY.md:1-2|note=[summary]\nrollout_summaries/foo.md:10-12|note=[details]\n</citation_entries>\n<rollout_ids>\n{first}\n{second}\n{first}\n</rollout_ids>"
|
||||
)];
|
||||
|
||||
let parsed = parse_memory_citation(citations).expect("memory citation should parse");
|
||||
|
||||
assert_eq!(
|
||||
parsed
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| (
|
||||
entry.path.clone(),
|
||||
entry.line_start,
|
||||
entry.line_end,
|
||||
entry.note.clone(),
|
||||
))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
("MEMORY.md".to_string(), 1, 2, "summary".to_string()),
|
||||
(
|
||||
"rollout_summaries/foo.md".to_string(),
|
||||
10,
|
||||
12,
|
||||
"details".to_string()
|
||||
),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
parsed.rollout_ids,
|
||||
vec![first.to_string(), second.to_string()]
|
||||
);
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
use std::path::Path;
|
||||
|
||||
pub async fn clear_memory_roots_contents(codex_home: &Path) -> std::io::Result<()> {
|
||||
for memory_root in [
|
||||
codex_home.join("memories"),
|
||||
codex_home.join("memories_extensions"),
|
||||
] {
|
||||
clear_memory_root_contents(memory_root.as_path()).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn clear_memory_root_contents(memory_root: &Path) -> std::io::Result<()> {
|
||||
match tokio::fs::symlink_metadata(memory_root).await {
|
||||
Ok(metadata) if metadata.file_type().is_symlink() => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!(
|
||||
"refusing to clear symlinked memory root {}",
|
||||
memory_root.display()
|
||||
),
|
||||
));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
tokio::fs::create_dir_all(memory_root).await?;
|
||||
|
||||
let mut entries = tokio::fs::read_dir(memory_root).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = entry.path();
|
||||
let file_type = entry.file_type().await?;
|
||||
if file_type.is_dir() {
|
||||
tokio::fs::remove_dir_all(path).await?;
|
||||
} else {
|
||||
tokio::fs::remove_file(path).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
use crate::memories::memory_extensions_root;
|
||||
use chrono::DateTime;
|
||||
use chrono::Duration;
|
||||
use chrono::NaiveDateTime;
|
||||
use chrono::Utc;
|
||||
use std::path::Path;
|
||||
use tracing::warn;
|
||||
|
||||
const FILENAME_TS_FORMAT: &str = "%Y-%m-%dT%H-%M-%S";
|
||||
pub(super) const EXTENSION_RESOURCE_RETENTION_DAYS: i64 = 7;
|
||||
|
||||
pub(super) async fn prune_old_extension_resources(memory_root: &Path) {
|
||||
prune_old_extension_resources_with_now(memory_root, Utc::now()).await
|
||||
}
|
||||
|
||||
async fn prune_old_extension_resources_with_now(memory_root: &Path, now: DateTime<Utc>) {
|
||||
let cutoff = now - Duration::days(EXTENSION_RESOURCE_RETENTION_DAYS);
|
||||
let extensions_root = memory_extensions_root(memory_root);
|
||||
let mut extensions = match tokio::fs::read_dir(&extensions_root).await {
|
||||
Ok(extensions) => extensions,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"failed reading memory extensions root {}: {err}",
|
||||
extensions_root.display()
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
while let Ok(Some(extension_entry)) = extensions.next_entry().await {
|
||||
let extension_path = extension_entry.path();
|
||||
let Ok(file_type) = extension_entry.file_type().await else {
|
||||
continue;
|
||||
};
|
||||
if !file_type.is_dir()
|
||||
|| !tokio::fs::try_exists(extension_path.join("instructions.md"))
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let resources_path = extension_path.join("resources");
|
||||
let mut resources = match tokio::fs::read_dir(&resources_path).await {
|
||||
Ok(resources) => resources,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"failed reading memory extension resources {}: {err}",
|
||||
resources_path.display()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
while let Ok(Some(resource_entry)) = resources.next_entry().await {
|
||||
let resource_file_path = resource_entry.path();
|
||||
let Ok(file_type) = resource_entry.file_type().await else {
|
||||
continue;
|
||||
};
|
||||
if !file_type.is_file() {
|
||||
continue;
|
||||
}
|
||||
let Some(file_name) = resource_file_path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !file_name.ends_with(".md") {
|
||||
continue;
|
||||
}
|
||||
let Some(resource_timestamp) = resource_timestamp(file_name) else {
|
||||
continue;
|
||||
};
|
||||
if resource_timestamp > cutoff {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Err(err) = tokio::fs::remove_file(&resource_file_path).await
|
||||
&& err.kind() != std::io::ErrorKind::NotFound
|
||||
{
|
||||
warn!(
|
||||
"failed pruning old memory extension resource {}: {err}",
|
||||
resource_file_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn resource_timestamp(file_name: &str) -> Option<DateTime<Utc>> {
|
||||
let timestamp = file_name.get(..19)?;
|
||||
let naive = NaiveDateTime::parse_from_str(timestamp, FILENAME_TS_FORMAT).ok()?;
|
||||
Some(DateTime::from_naive_utc_and_offset(naive, Utc))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "extensions_tests.rs"]
|
||||
mod tests;
|
||||
@@ -1,81 +0,0 @@
|
||||
use super::*;
|
||||
use pretty_assertions::assert_eq;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn prunes_only_old_resources_from_extensions_with_instructions() {
|
||||
let codex_home = TempDir::new().expect("create temp codex home");
|
||||
let memory_root = codex_home.path().join("memories");
|
||||
let extensions_root = memory_extensions_root(&memory_root);
|
||||
let chronicle_resources = extensions_root.join("chronicle/resources");
|
||||
tokio::fs::create_dir_all(&chronicle_resources)
|
||||
.await
|
||||
.expect("create chronicle resources");
|
||||
tokio::fs::write(
|
||||
extensions_root.join("chronicle/instructions.md"),
|
||||
"instructions",
|
||||
)
|
||||
.await
|
||||
.expect("write chronicle instructions");
|
||||
|
||||
let now = DateTime::from_naive_utc_and_offset(
|
||||
NaiveDateTime::parse_from_str("2026-04-14T12-00-00", FILENAME_TS_FORMAT)
|
||||
.expect("parse now"),
|
||||
Utc,
|
||||
);
|
||||
let old_file = chronicle_resources.join("2026-04-06T11-59-59-abcd-10min-old.md");
|
||||
let exact_cutoff_file = chronicle_resources.join("2026-04-07T12-00-00-abcd-10min-cutoff.md");
|
||||
let recent_file = chronicle_resources.join("2026-04-08T12-00-00-abcd-10min-recent.md");
|
||||
let invalid_file = chronicle_resources.join("not-a-timestamp.md");
|
||||
for file in [&old_file, &exact_cutoff_file, &recent_file, &invalid_file] {
|
||||
tokio::fs::write(file, "resource")
|
||||
.await
|
||||
.expect("write chronicle resource");
|
||||
}
|
||||
|
||||
let ignored_resources = extensions_root.join("ignored/resources");
|
||||
tokio::fs::create_dir_all(&ignored_resources)
|
||||
.await
|
||||
.expect("create ignored resources");
|
||||
let ignored_old_file = ignored_resources.join("2026-04-06T11-59-59-abcd-10min-old.md");
|
||||
tokio::fs::write(&ignored_old_file, "ignored")
|
||||
.await
|
||||
.expect("write ignored resource");
|
||||
|
||||
prune_old_extension_resources_with_now(&memory_root, now).await;
|
||||
|
||||
assert!(
|
||||
!tokio::fs::try_exists(&old_file)
|
||||
.await
|
||||
.expect("check old file")
|
||||
);
|
||||
assert!(
|
||||
!tokio::fs::try_exists(&exact_cutoff_file)
|
||||
.await
|
||||
.expect("check cutoff file")
|
||||
);
|
||||
assert!(
|
||||
tokio::fs::try_exists(&recent_file)
|
||||
.await
|
||||
.expect("check recent file")
|
||||
);
|
||||
assert!(
|
||||
tokio::fs::try_exists(&invalid_file)
|
||||
.await
|
||||
.expect("check invalid file")
|
||||
);
|
||||
assert!(
|
||||
tokio::fs::try_exists(&ignored_old_file)
|
||||
.await
|
||||
.expect("check ignored file")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_timestamp_prefix_from_resource_file_name() {
|
||||
let parsed = resource_timestamp("2026-04-06T11-59-59-abcd-10min-old.md")
|
||||
.expect("timestamp should parse");
|
||||
|
||||
assert_eq!(parsed.timestamp(), 1_775_476_799);
|
||||
assert!(resource_timestamp("not-a-timestamp.md").is_none());
|
||||
}
|
||||
@@ -1,37 +1,23 @@
|
||||
//! Memory subsystem for startup extraction and consolidation.
|
||||
//! Memory startup extraction and consolidation orchestration.
|
||||
//!
|
||||
//! The startup memory pipeline is split into two phases:
|
||||
//! - Phase 1: select rollouts, extract stage-1 raw memories, persist stage-1 outputs, and enqueue consolidation.
|
||||
//! - Phase 2: claim a global consolidation lock, materialize consolidation inputs, and dispatch one consolidation agent.
|
||||
|
||||
pub(crate) mod citations;
|
||||
mod control;
|
||||
mod extensions;
|
||||
mod phase1;
|
||||
mod phase2;
|
||||
pub(crate) mod prompts;
|
||||
mod start;
|
||||
mod storage;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
pub(crate) mod usage;
|
||||
mod workspace;
|
||||
|
||||
use codex_protocol::openai_models::ReasoningEffort;
|
||||
|
||||
pub use control::clear_memory_roots_contents;
|
||||
/// Starts the memory startup pipeline for eligible root sessions.
|
||||
/// This is the single entrypoint that `codex` uses to trigger memory startup.
|
||||
///
|
||||
/// This is the entry point to read and understand this module.
|
||||
pub(crate) use start::start_memories_startup_task;
|
||||
|
||||
mod artifacts {
|
||||
pub(super) const EXTENSIONS_SUBDIR: &str = "extensions";
|
||||
pub(super) const ROLLOUT_SUMMARIES_SUBDIR: &str = "rollout_summaries";
|
||||
pub(super) const RAW_MEMORIES_FILENAME: &str = "raw_memories.md";
|
||||
}
|
||||
|
||||
/// Phase 1 (startup extraction).
|
||||
mod phase_one {
|
||||
/// Default model used for phase 1.
|
||||
@@ -39,21 +25,9 @@ mod phase_one {
|
||||
/// Default reasoning effort used for phase 1.
|
||||
pub(super) const REASONING_EFFORT: super::ReasoningEffort = super::ReasoningEffort::Low;
|
||||
/// Prompt used for phase 1.
|
||||
pub(super) const PROMPT: &str = include_str!("../../templates/memories/stage_one_system.md");
|
||||
pub(super) const PROMPT: &str = codex_memories_write::STAGE_ONE_PROMPT;
|
||||
/// Concurrency cap for startup memory extraction and consolidation scheduling.
|
||||
pub(super) const CONCURRENCY_LIMIT: usize = 8;
|
||||
/// Fallback stage-1 rollout truncation limit (tokens) when model metadata
|
||||
/// does not include a valid context window.
|
||||
pub(super) const DEFAULT_STAGE_ONE_ROLLOUT_TOKEN_LIMIT: usize = 150_000;
|
||||
/// Maximum number of tokens from `memory_summary.md` injected into memory
|
||||
/// tool developer instructions.
|
||||
pub(super) const MEMORY_TOOL_DEVELOPER_INSTRUCTIONS_SUMMARY_TOKEN_LIMIT: usize = 5_000;
|
||||
/// Portion of the model effective input window reserved for the stage-1
|
||||
/// rollout input.
|
||||
///
|
||||
/// Keeping this below 100% leaves room for system instructions, prompt
|
||||
/// framing, and model output.
|
||||
pub(super) const CONTEXT_WINDOW_PERCENT: i64 = 70;
|
||||
/// Lease duration (seconds) for phase-1 job ownership.
|
||||
pub(super) const JOB_LEASE_SECONDS: i64 = 3_600;
|
||||
/// Backoff delay (seconds) before retrying a failed stage-1 extraction job.
|
||||
@@ -97,27 +71,3 @@ mod metrics {
|
||||
/// Histogram for aggregate token usage across one phase-2 consolidation run.
|
||||
pub(super) const MEMORY_PHASE_TWO_TOKEN_USAGE: &str = "codex.memory.phase2.token_usage";
|
||||
}
|
||||
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
pub fn memory_root(codex_home: &AbsolutePathBuf) -> AbsolutePathBuf {
|
||||
codex_home.join("memories")
|
||||
}
|
||||
|
||||
fn rollout_summaries_dir(root: &Path) -> PathBuf {
|
||||
root.join(artifacts::ROLLOUT_SUMMARIES_SUBDIR)
|
||||
}
|
||||
|
||||
fn memory_extensions_root(root: &Path) -> PathBuf {
|
||||
root.join(artifacts::EXTENSIONS_SUBDIR)
|
||||
}
|
||||
|
||||
fn raw_memories_file(root: &Path) -> PathBuf {
|
||||
root.join(artifacts::RAW_MEMORIES_FILENAME)
|
||||
}
|
||||
|
||||
async fn ensure_layout(root: &Path) -> std::io::Result<()> {
|
||||
tokio::fs::create_dir_all(rollout_summaries_dir(root)).await
|
||||
}
|
||||
|
||||
@@ -5,13 +5,13 @@ use crate::context::is_memory_excluded_contextual_user_fragment;
|
||||
use crate::memories::metrics;
|
||||
use crate::memories::phase_one;
|
||||
use crate::memories::phase_one::PRUNE_BATCH_SIZE;
|
||||
use crate::memories::prompts::build_stage_one_input_message;
|
||||
use crate::rollout::INTERACTIVE_SESSION_SOURCES;
|
||||
use crate::rollout::policy::should_persist_response_item_for_memories;
|
||||
use crate::session::session::Session;
|
||||
use crate::session::turn_context::TurnContext;
|
||||
use codex_api::ResponseEvent;
|
||||
use codex_config::types::MemoriesConfig;
|
||||
use codex_memories_write::build_stage_one_input_message;
|
||||
use codex_otel::SessionTelemetry;
|
||||
use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig;
|
||||
use codex_protocol::config_types::ServiceTier;
|
||||
|
||||
@@ -1,21 +1,21 @@
|
||||
use crate::agent::AgentStatus;
|
||||
use crate::agent::status::is_final as is_final_agent_status;
|
||||
use crate::config::Config;
|
||||
use crate::memories::extensions::prune_old_extension_resources;
|
||||
use crate::memories::memory_root;
|
||||
use crate::memories::metrics;
|
||||
use crate::memories::phase_two;
|
||||
use crate::memories::prompts::build_consolidation_prompt;
|
||||
use crate::memories::storage::rebuild_raw_memories_file_from_memories;
|
||||
use crate::memories::storage::sync_rollout_summaries_from_memories;
|
||||
use crate::memories::workspace::memory_workspace_diff;
|
||||
use crate::memories::workspace::prepare_memory_workspace;
|
||||
use crate::memories::workspace::reset_memory_workspace_baseline;
|
||||
use crate::memories::workspace::write_workspace_diff;
|
||||
use crate::session::emit_subagent_session_started;
|
||||
use crate::session::session::Session;
|
||||
use codex_config::Constrained;
|
||||
use codex_features::Feature;
|
||||
use codex_memories_write::build_consolidation_prompt;
|
||||
use codex_memories_write::memory_root;
|
||||
use codex_memories_write::prune_old_extension_resources;
|
||||
use codex_memories_write::rebuild_raw_memories_file_from_memories;
|
||||
use codex_memories_write::sync_rollout_summaries_from_memories;
|
||||
use codex_memories_write::workspace::memory_workspace_diff;
|
||||
use codex_memories_write::workspace::prepare_memory_workspace;
|
||||
use codex_memories_write::workspace::reset_memory_workspace_baseline;
|
||||
use codex_memories_write::workspace::write_workspace_diff;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::SandboxPolicy;
|
||||
|
||||
@@ -1,197 +0,0 @@
|
||||
use crate::memories::memory_extensions_root;
|
||||
use crate::memories::memory_root;
|
||||
use crate::memories::phase_one;
|
||||
use crate::memories::workspace::WORKSPACE_DIFF_FILENAME;
|
||||
use codex_protocol::openai_models::ModelInfo;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use codex_utils_output_truncation::TruncationPolicy;
|
||||
use codex_utils_output_truncation::truncate_text;
|
||||
use codex_utils_template::Template;
|
||||
use std::path::Path;
|
||||
use std::sync::LazyLock;
|
||||
use tokio::fs;
|
||||
use tracing::warn;
|
||||
|
||||
static CONSOLIDATION_PROMPT_TEMPLATE: LazyLock<Template> = LazyLock::new(|| {
|
||||
parse_embedded_template(
|
||||
include_str!("../../templates/memories/consolidation.md"),
|
||||
"memories/consolidation.md",
|
||||
)
|
||||
});
|
||||
static STAGE_ONE_INPUT_TEMPLATE: LazyLock<Template> = LazyLock::new(|| {
|
||||
parse_embedded_template(
|
||||
include_str!("../../templates/memories/stage_one_input.md"),
|
||||
"memories/stage_one_input.md",
|
||||
)
|
||||
});
|
||||
static MEMORY_TOOL_DEVELOPER_INSTRUCTIONS_TEMPLATE: LazyLock<Template> = LazyLock::new(|| {
|
||||
parse_embedded_template(
|
||||
include_str!("../../templates/memories/read_path.md"),
|
||||
"memories/read_path.md",
|
||||
)
|
||||
});
|
||||
static MEMORY_EXTENSIONS_FOLDER_STRUCTURE_TEMPLATE: LazyLock<Template> = LazyLock::new(|| {
|
||||
parse_embedded_template(
|
||||
MEMORY_EXTENSIONS_FOLDER_STRUCTURE,
|
||||
"memories/extensions_folder_structure.md",
|
||||
)
|
||||
});
|
||||
static MEMORY_EXTENSIONS_PRIMARY_INPUTS_TEMPLATE: LazyLock<Template> = LazyLock::new(|| {
|
||||
parse_embedded_template(
|
||||
MEMORY_EXTENSIONS_PRIMARY_INPUTS,
|
||||
"memories/extensions_primary_inputs.md",
|
||||
)
|
||||
});
|
||||
|
||||
fn parse_embedded_template(source: &'static str, template_name: &str) -> Template {
|
||||
match Template::parse(source) {
|
||||
Ok(template) => template,
|
||||
Err(err) => panic!("embedded template {template_name} is invalid: {err}"),
|
||||
}
|
||||
}
|
||||
|
||||
const MEMORY_EXTENSIONS_FOLDER_STRUCTURE: &str = r#"
|
||||
Memory extensions (under {{ memory_extensions_root }}/):
|
||||
|
||||
- <extension_name>/instructions.md
|
||||
- Source-specific guidance for interpreting additional memory signals. If an
|
||||
extension folder exists, you must read its instructions.md to determine how to use this memory
|
||||
source.
|
||||
|
||||
If the user has any memory extensions, you MUST read the instructions for each extension to
|
||||
determine how to use the memory source. If the workspace diff shows deleted extension resource files,
|
||||
remove stale memories derived only from those resources. If it has no extension folders, continue
|
||||
with the standard memory inputs only.
|
||||
"#;
|
||||
|
||||
const MEMORY_EXTENSIONS_PRIMARY_INPUTS: &str = r#"
|
||||
Optional source-specific inputs:
|
||||
Under `{{ memory_extensions_root }}/`:
|
||||
|
||||
- `<extension_name>/instructions.md`
|
||||
- If extension folders exist, read each instructions.md first and follow it when interpreting
|
||||
that extension's memory source.
|
||||
|
||||
If the workspace diff shows deleted memory extension resources, use that extension-specific deletion
|
||||
signal to remove stale memories derived only from those resources.
|
||||
"#;
|
||||
|
||||
/// Builds the consolidation subagent prompt for a specific memory root.
|
||||
pub(super) fn build_consolidation_prompt(memory_root: &Path) -> String {
|
||||
let memory_extensions_root = memory_extensions_root(memory_root);
|
||||
let memory_extensions_exist = memory_extensions_root.is_dir();
|
||||
let memory_root = memory_root.display().to_string();
|
||||
let memory_extensions_root = memory_extensions_root.display().to_string();
|
||||
let phase2_workspace_diff_file = WORKSPACE_DIFF_FILENAME.to_string();
|
||||
let memory_extensions_folder_structure = if memory_extensions_exist {
|
||||
render_memory_extensions_block(
|
||||
&MEMORY_EXTENSIONS_FOLDER_STRUCTURE_TEMPLATE,
|
||||
&memory_extensions_root,
|
||||
)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
let memory_extensions_primary_inputs = if memory_extensions_exist {
|
||||
render_memory_extensions_block(
|
||||
&MEMORY_EXTENSIONS_PRIMARY_INPUTS_TEMPLATE,
|
||||
&memory_extensions_root,
|
||||
)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
CONSOLIDATION_PROMPT_TEMPLATE
|
||||
.render([
|
||||
("memory_root", memory_root.as_str()),
|
||||
(
|
||||
"memory_extensions_folder_structure",
|
||||
memory_extensions_folder_structure.as_str(),
|
||||
),
|
||||
(
|
||||
"memory_extensions_primary_inputs",
|
||||
memory_extensions_primary_inputs.as_str(),
|
||||
),
|
||||
(
|
||||
"phase2_workspace_diff_file",
|
||||
phase2_workspace_diff_file.as_str(),
|
||||
),
|
||||
])
|
||||
.unwrap_or_else(|err| {
|
||||
warn!("failed to render memories consolidation prompt template: {err}");
|
||||
format!(
|
||||
"## Memory Phase 2 (Consolidation)\nConsolidate Codex memories in: {memory_root}\n\nRead {phase2_workspace_diff_file} first."
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn render_memory_extensions_block(template: &Template, memory_extensions_root: &str) -> String {
|
||||
template
|
||||
.render([("memory_extensions_root", memory_extensions_root)])
|
||||
.unwrap_or_else(|err| {
|
||||
warn!("failed to render memories extension prompt block: {err}");
|
||||
String::new()
|
||||
})
|
||||
}
|
||||
|
||||
/// Builds the stage-1 user message containing rollout metadata and content.
|
||||
///
|
||||
/// Large rollout payloads are truncated to 70% of the active model's effective
|
||||
/// input window token budget while keeping both head and tail context.
|
||||
pub(super) fn build_stage_one_input_message(
|
||||
model_info: &ModelInfo,
|
||||
rollout_path: &Path,
|
||||
rollout_cwd: &Path,
|
||||
rollout_contents: &str,
|
||||
) -> anyhow::Result<String> {
|
||||
let rollout_token_limit = model_info
|
||||
.resolved_context_window()
|
||||
.and_then(|limit| (limit > 0).then_some(limit))
|
||||
.map(|limit| limit.saturating_mul(model_info.effective_context_window_percent) / 100)
|
||||
.map(|limit| (limit.saturating_mul(phase_one::CONTEXT_WINDOW_PERCENT) / 100).max(1))
|
||||
.and_then(|limit| usize::try_from(limit).ok())
|
||||
.unwrap_or(phase_one::DEFAULT_STAGE_ONE_ROLLOUT_TOKEN_LIMIT);
|
||||
let truncated_rollout_contents = truncate_text(
|
||||
rollout_contents,
|
||||
TruncationPolicy::Tokens(rollout_token_limit),
|
||||
);
|
||||
|
||||
let rollout_path = rollout_path.display().to_string();
|
||||
let rollout_cwd = rollout_cwd.display().to_string();
|
||||
Ok(STAGE_ONE_INPUT_TEMPLATE.render([
|
||||
("rollout_path", rollout_path.as_str()),
|
||||
("rollout_cwd", rollout_cwd.as_str()),
|
||||
("rollout_contents", truncated_rollout_contents.as_str()),
|
||||
])?)
|
||||
}
|
||||
|
||||
/// Build prompt used for read path. This prompt must be added to the developer instructions. In
|
||||
/// case of large memory files, the `memory_summary.md` is truncated at
|
||||
/// [phase_one::MEMORY_TOOL_DEVELOPER_INSTRUCTIONS_SUMMARY_TOKEN_LIMIT].
|
||||
pub(crate) async fn build_memory_tool_developer_instructions(
|
||||
codex_home: &AbsolutePathBuf,
|
||||
) -> Option<String> {
|
||||
let base_path = memory_root(codex_home);
|
||||
let memory_summary_path = base_path.join("memory_summary.md");
|
||||
let memory_summary = fs::read_to_string(&memory_summary_path)
|
||||
.await
|
||||
.ok()?
|
||||
.trim()
|
||||
.to_string();
|
||||
let memory_summary = truncate_text(
|
||||
&memory_summary,
|
||||
TruncationPolicy::Tokens(phase_one::MEMORY_TOOL_DEVELOPER_INSTRUCTIONS_SUMMARY_TOKEN_LIMIT),
|
||||
);
|
||||
if memory_summary.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let base_path = base_path.display().to_string();
|
||||
MEMORY_TOOL_DEVELOPER_INSTRUCTIONS_TEMPLATE
|
||||
.render([
|
||||
("base_path", base_path.as_str()),
|
||||
("memory_summary", memory_summary.as_str()),
|
||||
])
|
||||
.ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "prompts_tests.rs"]
|
||||
mod tests;
|
||||
@@ -1,104 +0,0 @@
|
||||
use super::*;
|
||||
use codex_models_manager::model_info::model_info_from_slug;
|
||||
use core_test_support::PathExt;
|
||||
use pretty_assertions::assert_eq;
|
||||
use tempfile::tempdir;
|
||||
use tokio::fs as tokio_fs;
|
||||
|
||||
#[test]
|
||||
fn build_stage_one_input_message_truncates_rollout_using_model_context_window() {
|
||||
let input = format!("{}{}{}", "a".repeat(700_000), "middle", "z".repeat(700_000));
|
||||
let mut model_info = model_info_from_slug("gpt-5.3-codex");
|
||||
model_info.context_window = Some(123_000);
|
||||
let expected_rollout_token_limit = usize::try_from(
|
||||
((123_000_i64 * model_info.effective_context_window_percent) / 100)
|
||||
* phase_one::CONTEXT_WINDOW_PERCENT
|
||||
/ 100,
|
||||
)
|
||||
.unwrap();
|
||||
let expected_truncated = truncate_text(
|
||||
&input,
|
||||
TruncationPolicy::Tokens(expected_rollout_token_limit),
|
||||
);
|
||||
let message = build_stage_one_input_message(
|
||||
&model_info,
|
||||
Path::new("/tmp/rollout.jsonl"),
|
||||
Path::new("/tmp"),
|
||||
&input,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(expected_truncated.contains("tokens truncated"));
|
||||
assert!(expected_truncated.starts_with('a'));
|
||||
assert!(expected_truncated.ends_with('z'));
|
||||
assert!(message.contains(&expected_truncated));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_stage_one_input_message_uses_default_limit_when_model_context_window_missing() {
|
||||
let input = format!("{}{}{}", "a".repeat(700_000), "middle", "z".repeat(700_000));
|
||||
let mut model_info = model_info_from_slug("gpt-5.3-codex");
|
||||
model_info.context_window = None;
|
||||
model_info.max_context_window = None;
|
||||
let expected_truncated = truncate_text(
|
||||
&input,
|
||||
TruncationPolicy::Tokens(phase_one::DEFAULT_STAGE_ONE_ROLLOUT_TOKEN_LIMIT),
|
||||
);
|
||||
let message = build_stage_one_input_message(
|
||||
&model_info,
|
||||
Path::new("/tmp/rollout.jsonl"),
|
||||
Path::new("/tmp"),
|
||||
&input,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(message.contains(&expected_truncated));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_consolidation_prompt_points_to_workspace_diff_and_extension_tree() {
|
||||
let temp = tempdir().unwrap();
|
||||
let memory_root = temp.path().join("memories");
|
||||
let memory_extensions_root = memory_root.join("extensions");
|
||||
std::fs::create_dir_all(&memory_extensions_root).unwrap();
|
||||
|
||||
let prompt = build_consolidation_prompt(&memory_root);
|
||||
|
||||
assert!(prompt.contains("Memory workspace diff:"));
|
||||
assert!(prompt.contains("phase2_workspace_diff.md"));
|
||||
assert!(prompt.contains(&format!(
|
||||
"Memory extensions (under {}/):",
|
||||
memory_extensions_root.display()
|
||||
)));
|
||||
assert!(prompt.contains("workspace diff shows deleted extension resource files"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_memory_tool_developer_instructions_renders_embedded_template() {
|
||||
let temp = tempdir().unwrap();
|
||||
let codex_home = temp.path().abs();
|
||||
let memories_dir = codex_home.join("memories");
|
||||
tokio_fs::create_dir_all(&memories_dir).await.unwrap();
|
||||
tokio_fs::write(
|
||||
memories_dir.join("memory_summary.md"),
|
||||
"Short memory summary for tests.",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let instructions = build_memory_tool_developer_instructions(&codex_home)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(instructions.contains(&format!(
|
||||
"- {}/memory_summary.md (already provided below; do NOT open again)",
|
||||
memories_dir.display()
|
||||
)));
|
||||
assert!(instructions.contains("Short memory summary for tests."));
|
||||
assert_eq!(
|
||||
instructions
|
||||
.matches("========= MEMORY_SUMMARY BEGINS =========")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
}
|
||||
@@ -1,242 +0,0 @@
|
||||
use codex_state::Stage1Output;
|
||||
use std::collections::HashSet;
|
||||
use std::fmt::Write as _;
|
||||
use std::path::Path;
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::memories::ensure_layout;
|
||||
use crate::memories::raw_memories_file;
|
||||
use crate::memories::rollout_summaries_dir;
|
||||
|
||||
/// Rebuild `raw_memories.md` from DB-backed stage-1 outputs.
|
||||
pub(super) async fn rebuild_raw_memories_file_from_memories(
|
||||
root: &Path,
|
||||
memories: &[Stage1Output],
|
||||
max_raw_memories_for_consolidation: usize,
|
||||
) -> std::io::Result<()> {
|
||||
ensure_layout(root).await?;
|
||||
rebuild_raw_memories_file(root, memories, max_raw_memories_for_consolidation).await
|
||||
}
|
||||
|
||||
/// Syncs canonical rollout summary files from DB-backed stage-1 output rows.
|
||||
pub(super) async fn sync_rollout_summaries_from_memories(
|
||||
root: &Path,
|
||||
memories: &[Stage1Output],
|
||||
max_raw_memories_for_consolidation: usize,
|
||||
) -> std::io::Result<()> {
|
||||
ensure_layout(root).await?;
|
||||
|
||||
let retained = retained_memories(memories, max_raw_memories_for_consolidation);
|
||||
let keep = retained
|
||||
.iter()
|
||||
.map(rollout_summary_file_stem)
|
||||
.collect::<HashSet<_>>();
|
||||
prune_rollout_summaries(root, &keep).await?;
|
||||
|
||||
for memory in retained {
|
||||
write_rollout_summary_for_thread(root, memory).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn rebuild_raw_memories_file(
|
||||
root: &Path,
|
||||
memories: &[Stage1Output],
|
||||
max_raw_memories_for_consolidation: usize,
|
||||
) -> std::io::Result<()> {
|
||||
let retained = retained_memories(memories, max_raw_memories_for_consolidation);
|
||||
let mut body = String::from("# Raw Memories\n\n");
|
||||
|
||||
if retained.is_empty() {
|
||||
body.push_str("No raw memories yet.\n");
|
||||
return tokio::fs::write(raw_memories_file(root), body).await;
|
||||
}
|
||||
|
||||
body.push_str("Merged stage-1 raw memories (latest first):\n\n");
|
||||
for memory in retained {
|
||||
writeln!(body, "## Thread `{}`", memory.thread_id).map_err(raw_memories_format_error)?;
|
||||
writeln!(
|
||||
body,
|
||||
"updated_at: {}",
|
||||
memory.source_updated_at.to_rfc3339()
|
||||
)
|
||||
.map_err(raw_memories_format_error)?;
|
||||
writeln!(body, "cwd: {}", memory.cwd.display()).map_err(raw_memories_format_error)?;
|
||||
writeln!(body, "rollout_path: {}", memory.rollout_path.display())
|
||||
.map_err(raw_memories_format_error)?;
|
||||
let rollout_summary_file = format!("{}.md", rollout_summary_file_stem(memory));
|
||||
writeln!(body, "rollout_summary_file: {rollout_summary_file}")
|
||||
.map_err(raw_memories_format_error)?;
|
||||
writeln!(body).map_err(raw_memories_format_error)?;
|
||||
body.push_str(memory.raw_memory.trim());
|
||||
body.push_str("\n\n");
|
||||
}
|
||||
|
||||
tokio::fs::write(raw_memories_file(root), body).await
|
||||
}
|
||||
|
||||
async fn prune_rollout_summaries(root: &Path, keep: &HashSet<String>) -> std::io::Result<()> {
|
||||
let dir_path = rollout_summaries_dir(root);
|
||||
let mut dir = match tokio::fs::read_dir(&dir_path).await {
|
||||
Ok(dir) => dir,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
while let Some(entry) = dir.next_entry().await? {
|
||||
let path = entry.path();
|
||||
let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
let Some(stem) = file_name.strip_suffix(".md") else {
|
||||
continue;
|
||||
};
|
||||
if !keep.contains(stem)
|
||||
&& let Err(err) = tokio::fs::remove_file(&path).await
|
||||
&& err.kind() != std::io::ErrorKind::NotFound
|
||||
{
|
||||
warn!(
|
||||
"failed pruning outdated rollout summary {}: {err}",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_rollout_summary_for_thread(
|
||||
root: &Path,
|
||||
memory: &Stage1Output,
|
||||
) -> std::io::Result<()> {
|
||||
let file_stem = rollout_summary_file_stem(memory);
|
||||
let path = rollout_summaries_dir(root).join(format!("{file_stem}.md"));
|
||||
|
||||
let mut body = String::new();
|
||||
writeln!(body, "thread_id: {}", memory.thread_id).map_err(rollout_summary_format_error)?;
|
||||
writeln!(
|
||||
body,
|
||||
"updated_at: {}",
|
||||
memory.source_updated_at.to_rfc3339()
|
||||
)
|
||||
.map_err(rollout_summary_format_error)?;
|
||||
writeln!(body, "rollout_path: {}", memory.rollout_path.display())
|
||||
.map_err(rollout_summary_format_error)?;
|
||||
writeln!(body, "cwd: {}", memory.cwd.display()).map_err(rollout_summary_format_error)?;
|
||||
if let Some(git_branch) = memory.git_branch.as_deref() {
|
||||
writeln!(body, "git_branch: {git_branch}").map_err(rollout_summary_format_error)?;
|
||||
}
|
||||
writeln!(body).map_err(rollout_summary_format_error)?;
|
||||
body.push_str(&memory.rollout_summary);
|
||||
body.push('\n');
|
||||
|
||||
tokio::fs::write(path, body).await
|
||||
}
|
||||
|
||||
fn retained_memories(
|
||||
memories: &[Stage1Output],
|
||||
max_raw_memories_for_consolidation: usize,
|
||||
) -> &[Stage1Output] {
|
||||
&memories[..memories.len().min(max_raw_memories_for_consolidation)]
|
||||
}
|
||||
|
||||
fn raw_memories_format_error(err: std::fmt::Error) -> std::io::Error {
|
||||
std::io::Error::other(format!("format raw memories: {err}"))
|
||||
}
|
||||
|
||||
fn rollout_summary_format_error(err: std::fmt::Error) -> std::io::Error {
|
||||
std::io::Error::other(format!("format rollout summary: {err}"))
|
||||
}
|
||||
|
||||
pub(crate) fn rollout_summary_file_stem(memory: &Stage1Output) -> String {
|
||||
rollout_summary_file_stem_from_parts(
|
||||
memory.thread_id,
|
||||
memory.source_updated_at,
|
||||
memory.rollout_slug.as_deref(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn rollout_summary_file_stem_from_parts(
|
||||
thread_id: codex_protocol::ThreadId,
|
||||
source_updated_at: chrono::DateTime<chrono::Utc>,
|
||||
rollout_slug: Option<&str>,
|
||||
) -> String {
|
||||
const ROLLOUT_SLUG_MAX_LEN: usize = 60;
|
||||
const SHORT_HASH_ALPHABET: &[u8; 62] =
|
||||
b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
|
||||
const SHORT_HASH_SPACE: u32 = 14_776_336;
|
||||
|
||||
let thread_id = thread_id.to_string();
|
||||
let (timestamp_fragment, short_hash_seed) = match Uuid::parse_str(&thread_id) {
|
||||
Ok(thread_uuid) => {
|
||||
let timestamp = thread_uuid
|
||||
.get_timestamp()
|
||||
.and_then(|uuid_timestamp| {
|
||||
let (seconds, nanos) = uuid_timestamp.to_unix();
|
||||
i64::try_from(seconds).ok().and_then(|secs| {
|
||||
chrono::DateTime::<chrono::Utc>::from_timestamp(secs, nanos)
|
||||
})
|
||||
})
|
||||
.unwrap_or(source_updated_at);
|
||||
let short_hash_seed = (thread_uuid.as_u128() & 0xFFFF_FFFF) as u32;
|
||||
(
|
||||
timestamp.format("%Y-%m-%dT%H-%M-%S").to_string(),
|
||||
short_hash_seed,
|
||||
)
|
||||
}
|
||||
Err(_) => {
|
||||
let mut short_hash_seed = 0u32;
|
||||
for byte in thread_id.bytes() {
|
||||
short_hash_seed = short_hash_seed
|
||||
.wrapping_mul(31)
|
||||
.wrapping_add(u32::from(byte));
|
||||
}
|
||||
(
|
||||
source_updated_at.format("%Y-%m-%dT%H-%M-%S").to_string(),
|
||||
short_hash_seed,
|
||||
)
|
||||
}
|
||||
};
|
||||
let mut short_hash_value = short_hash_seed % SHORT_HASH_SPACE;
|
||||
let mut short_hash_chars = ['0'; 4];
|
||||
for idx in (0..short_hash_chars.len()).rev() {
|
||||
let alphabet_idx = (short_hash_value % SHORT_HASH_ALPHABET.len() as u32) as usize;
|
||||
short_hash_chars[idx] = SHORT_HASH_ALPHABET[alphabet_idx] as char;
|
||||
short_hash_value /= SHORT_HASH_ALPHABET.len() as u32;
|
||||
}
|
||||
let short_hash: String = short_hash_chars.iter().collect();
|
||||
let file_prefix = format!("{timestamp_fragment}-{short_hash}");
|
||||
|
||||
let Some(raw_slug) = rollout_slug else {
|
||||
return file_prefix;
|
||||
};
|
||||
|
||||
let mut slug = String::with_capacity(ROLLOUT_SLUG_MAX_LEN);
|
||||
for ch in raw_slug.chars() {
|
||||
if slug.len() >= ROLLOUT_SLUG_MAX_LEN {
|
||||
break;
|
||||
}
|
||||
|
||||
if ch.is_ascii_alphanumeric() {
|
||||
slug.push(ch.to_ascii_lowercase());
|
||||
} else {
|
||||
slug.push('_');
|
||||
}
|
||||
}
|
||||
|
||||
while slug.ends_with('_') {
|
||||
slug.pop();
|
||||
}
|
||||
|
||||
if slug.is_empty() {
|
||||
file_prefix
|
||||
} else {
|
||||
format!("{file_prefix}-{slug}")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "storage_tests.rs"]
|
||||
mod tests;
|
||||
@@ -1,70 +0,0 @@
|
||||
use super::rollout_summary_file_stem;
|
||||
use super::rollout_summary_file_stem_from_parts;
|
||||
use chrono::TimeZone;
|
||||
use chrono::Utc;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_state::Stage1Output;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::path::PathBuf;
|
||||
const FIXED_PREFIX: &str = "2025-02-11T15-35-19-jqmb";
|
||||
|
||||
fn stage1_output_with_slug(thread_id: ThreadId, rollout_slug: Option<&str>) -> Stage1Output {
|
||||
Stage1Output {
|
||||
thread_id,
|
||||
source_updated_at: Utc.timestamp_opt(123, 0).single().expect("timestamp"),
|
||||
raw_memory: "raw memory".to_string(),
|
||||
rollout_summary: "summary".to_string(),
|
||||
rollout_slug: rollout_slug.map(ToString::to_string),
|
||||
rollout_path: PathBuf::from("/tmp/rollout.jsonl"),
|
||||
cwd: PathBuf::from("/tmp/workspace"),
|
||||
git_branch: None,
|
||||
generated_at: Utc.timestamp_opt(124, 0).single().expect("timestamp"),
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_thread_id() -> ThreadId {
|
||||
ThreadId::try_from("0194f5a6-89ab-7cde-8123-456789abcdef").expect("valid thread id")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rollout_summary_file_stem_uses_uuid_timestamp_and_hash_when_slug_missing() {
|
||||
let thread_id = fixed_thread_id();
|
||||
let memory = stage1_output_with_slug(thread_id, /*rollout_slug*/ None);
|
||||
|
||||
assert_eq!(rollout_summary_file_stem(&memory), FIXED_PREFIX);
|
||||
assert_eq!(
|
||||
rollout_summary_file_stem_from_parts(
|
||||
memory.thread_id,
|
||||
memory.source_updated_at,
|
||||
memory.rollout_slug.as_deref(),
|
||||
),
|
||||
FIXED_PREFIX
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rollout_summary_file_stem_sanitizes_and_truncates_slug() {
|
||||
let thread_id = fixed_thread_id();
|
||||
let memory = stage1_output_with_slug(
|
||||
thread_id,
|
||||
Some("Unsafe Slug/With Spaces & Symbols + EXTRA_LONG_12345_67890_ABCDE_fghij_klmno"),
|
||||
);
|
||||
|
||||
let stem = rollout_summary_file_stem(&memory);
|
||||
let slug = stem
|
||||
.strip_prefix(&format!("{FIXED_PREFIX}-"))
|
||||
.expect("slug suffix should be present");
|
||||
assert_eq!(slug.len(), 60);
|
||||
assert_eq!(
|
||||
slug,
|
||||
"unsafe_slug_with_spaces___symbols___extra_long_12345_67890_a"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rollout_summary_file_stem_uses_uuid_timestamp_and_hash_when_slug_is_empty() {
|
||||
let thread_id = fixed_thread_id();
|
||||
let memory = stage1_output_with_slug(thread_id, Some(""));
|
||||
|
||||
assert_eq!(rollout_summary_file_stem(&memory), FIXED_PREFIX);
|
||||
}
|
||||
@@ -1,13 +1,13 @@
|
||||
use super::control::clear_memory_root_contents;
|
||||
use super::storage::rebuild_raw_memories_file_from_memories;
|
||||
use super::storage::sync_rollout_summaries_from_memories;
|
||||
use crate::memories::ensure_layout;
|
||||
use crate::memories::memory_root;
|
||||
use crate::memories::raw_memories_file;
|
||||
use crate::memories::rollout_summaries_dir;
|
||||
use chrono::TimeZone;
|
||||
use chrono::Utc;
|
||||
use codex_config::types::DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION;
|
||||
use codex_memories_write::clear_memory_roots_contents;
|
||||
use codex_memories_write::ensure_layout;
|
||||
use codex_memories_write::memory_root;
|
||||
use codex_memories_write::raw_memories_file;
|
||||
use codex_memories_write::rebuild_raw_memories_file_from_memories;
|
||||
use codex_memories_write::rollout_summaries_dir;
|
||||
use codex_memories_write::sync_rollout_summaries_from_memories;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_state::Stage1Output;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
@@ -68,7 +68,7 @@ fn stage_one_output_schema_requires_rollout_slug_and_keeps_it_nullable() {
|
||||
#[tokio::test]
|
||||
async fn clear_memory_root_contents_preserves_root_directory() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let root = dir.path().join("memory");
|
||||
let root = dir.path().join("memories");
|
||||
let nested_dir = root.join("rollout_summaries");
|
||||
tokio::fs::create_dir_all(&nested_dir)
|
||||
.await
|
||||
@@ -80,7 +80,7 @@ async fn clear_memory_root_contents_preserves_root_directory() {
|
||||
.await
|
||||
.expect("write rollout summary");
|
||||
|
||||
clear_memory_root_contents(&root)
|
||||
clear_memory_roots_contents(dir.path())
|
||||
.await
|
||||
.expect("clear memory root contents");
|
||||
|
||||
@@ -116,10 +116,10 @@ async fn clear_memory_root_contents_rejects_symlinked_root() {
|
||||
.await
|
||||
.expect("write target file");
|
||||
|
||||
let root = dir.path().join("memory");
|
||||
let root = dir.path().join("memories");
|
||||
std::os::unix::fs::symlink(&target, &root).expect("create memory root symlink");
|
||||
|
||||
let err = clear_memory_root_contents(&root)
|
||||
let err = clear_memory_roots_contents(dir.path())
|
||||
.await
|
||||
.expect_err("symlinked memory root should be rejected");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
|
||||
@@ -509,13 +509,7 @@ mod phase2 {
|
||||
use crate::agent::AgentControl;
|
||||
use crate::config::Config;
|
||||
use crate::config::test_config;
|
||||
use crate::memories::memory_root;
|
||||
use crate::memories::phase2;
|
||||
use crate::memories::raw_memories_file;
|
||||
use crate::memories::rollout_summaries_dir;
|
||||
use crate::memories::storage::rebuild_raw_memories_file_from_memories;
|
||||
use crate::memories::storage::sync_rollout_summaries_from_memories;
|
||||
use crate::memories::workspace::prepare_memory_workspace;
|
||||
use crate::session::session::Session;
|
||||
use crate::session::tests::make_session_and_context;
|
||||
use chrono::Duration as ChronoDuration;
|
||||
@@ -524,6 +518,12 @@ mod phase2 {
|
||||
use codex_config::types::McpServerConfig;
|
||||
use codex_features::Feature;
|
||||
use codex_login::CodexAuth;
|
||||
use codex_memories_write::memory_root;
|
||||
use codex_memories_write::raw_memories_file;
|
||||
use codex_memories_write::rebuild_raw_memories_file_from_memories;
|
||||
use codex_memories_write::rollout_summaries_dir;
|
||||
use codex_memories_write::sync_rollout_summaries_from_memories;
|
||||
use codex_memories_write::workspace::prepare_memory_workspace;
|
||||
use codex_protocol::AgentPath;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
use anyhow::Context;
|
||||
use codex_git_utils::GitBaselineDiff;
|
||||
use codex_git_utils::diff_since_latest_init;
|
||||
use codex_git_utils::ensure_git_baseline_repository;
|
||||
use codex_git_utils::reset_git_repository;
|
||||
use std::path::Path;
|
||||
|
||||
/// Generated diff file the Phase 2 consolidation agent reads before editing memories.
|
||||
pub(super) const WORKSPACE_DIFF_FILENAME: &str = "phase2_workspace_diff.md";
|
||||
|
||||
const WORKSPACE_DIFF_MAX_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Prepares the memory directory for git-baseline diffing.
|
||||
///
|
||||
/// This keeps an existing usable `.git/` baseline intact. It initializes a new git baseline when the
|
||||
/// metadata is missing or unusable, and removes any stale generated `phase2_workspace_diff.md` file
|
||||
/// so that the next diff does not include a previous prompt artifact.
|
||||
pub(super) async fn prepare_memory_workspace(root: &Path) -> anyhow::Result<()> {
|
||||
tokio::fs::create_dir_all(root)
|
||||
.await
|
||||
.with_context(|| format!("create memory workspace {}", root.display()))?;
|
||||
remove_workspace_diff(root).await?;
|
||||
ensure_git_baseline_repository(root).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the current workspace diff after removing any stale generated diff artifact.
|
||||
///
|
||||
/// The removed file is only `phase2_workspace_diff.md`; memory artifacts and `.git/` metadata are
|
||||
/// left intact.
|
||||
pub(super) async fn memory_workspace_diff(root: &Path) -> anyhow::Result<GitBaselineDiff> {
|
||||
remove_workspace_diff(root).await?;
|
||||
diff_since_latest_init(root).await
|
||||
}
|
||||
|
||||
/// Writes `phase2_workspace_diff.md` with a bounded git-style diff from the current baseline.
|
||||
pub(super) async fn write_workspace_diff(
|
||||
root: &Path,
|
||||
diff: &GitBaselineDiff,
|
||||
) -> anyhow::Result<()> {
|
||||
let path = root.join(WORKSPACE_DIFF_FILENAME);
|
||||
tokio::fs::write(&path, render_workspace_diff_file(diff))
|
||||
.await
|
||||
.with_context(|| format!("write memory workspace diff file {}", path.display()))
|
||||
}
|
||||
|
||||
/// Marks the current memory root as the new baseline.
|
||||
///
|
||||
/// The generated diff file is removed before resetting the baseline so deleted memory content is
|
||||
/// not retained in the prompt artifact or in unreachable git objects.
|
||||
pub(super) async fn reset_memory_workspace_baseline(root: &Path) -> anyhow::Result<()> {
|
||||
remove_workspace_diff(root).await?;
|
||||
reset_git_repository(root).await
|
||||
}
|
||||
|
||||
/// Removes the generated `phase2_workspace_diff.md` prompt artifact.
|
||||
///
|
||||
/// This does not remove `.git/`, reset the baseline, or delete memory content. It is used before
|
||||
/// diffing and before baseline reset so the generated diff file itself is not treated as memory
|
||||
/// workspace input.
|
||||
pub(super) async fn remove_workspace_diff(root: &Path) -> anyhow::Result<()> {
|
||||
let path = root.join(WORKSPACE_DIFF_FILENAME);
|
||||
match tokio::fs::remove_file(&path).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(err)
|
||||
.with_context(|| format!("remove memory workspace diff file {}", path.display())),
|
||||
}
|
||||
}
|
||||
|
||||
fn render_workspace_diff_file(diff: &GitBaselineDiff) -> String {
|
||||
let mut rendered = String::from(
|
||||
"# Memory Workspace Diff\n\n\
|
||||
Generated by Codex before Phase 2 memory consolidation. Read this file first and do not edit it.\n\n\
|
||||
## Status\n",
|
||||
);
|
||||
|
||||
if !diff.has_changes() {
|
||||
rendered.push_str("- none\n");
|
||||
return rendered;
|
||||
}
|
||||
|
||||
for change in &diff.changes {
|
||||
rendered.push_str(&format!("- {} {}\n", change.status.label(), change.path));
|
||||
}
|
||||
rendered.push_str("\n## Diff\n\n```diff\n");
|
||||
append_bounded_diff(&mut rendered, &diff.unified_diff);
|
||||
rendered.push_str("```\n");
|
||||
rendered
|
||||
}
|
||||
|
||||
fn append_bounded_diff(rendered: &mut String, diff: &str) {
|
||||
if diff.len() <= WORKSPACE_DIFF_MAX_BYTES {
|
||||
rendered.push_str(diff);
|
||||
if !diff.ends_with('\n') {
|
||||
rendered.push('\n');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let boundary = previous_char_boundary(diff, WORKSPACE_DIFF_MAX_BYTES);
|
||||
rendered.push_str(&diff[..boundary]);
|
||||
if !rendered.ends_with('\n') {
|
||||
rendered.push('\n');
|
||||
}
|
||||
rendered.push_str(&format!(
|
||||
"\n[workspace diff truncated at {WORKSPACE_DIFF_MAX_BYTES} bytes]\n"
|
||||
));
|
||||
}
|
||||
|
||||
fn previous_char_boundary(value: &str, max_bytes: usize) -> usize {
|
||||
if max_bytes >= value.len() {
|
||||
return value.len();
|
||||
}
|
||||
let mut index = max_bytes;
|
||||
while !value.is_char_boundary(index) {
|
||||
index -= 1;
|
||||
}
|
||||
index
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "workspace_tests.rs"]
|
||||
mod tests;
|
||||
@@ -1,78 +0,0 @@
|
||||
use super::*;
|
||||
use codex_git_utils::GitBaselineChange;
|
||||
use codex_git_utils::GitBaselineChangeStatus;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::fs;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn render_workspace_diff_file_bounds_large_diff() {
|
||||
let diff = GitBaselineDiff {
|
||||
changes: vec![GitBaselineChange {
|
||||
status: GitBaselineChangeStatus::Modified,
|
||||
path: "MEMORY.md".to_string(),
|
||||
}],
|
||||
unified_diff: "a".repeat(WORKSPACE_DIFF_MAX_BYTES + 128),
|
||||
};
|
||||
|
||||
let rendered = render_workspace_diff_file(&diff);
|
||||
|
||||
assert!(rendered.contains("- M MEMORY.md"));
|
||||
assert!(rendered.contains("[workspace diff truncated at 4194304 bytes]"));
|
||||
assert!(rendered.ends_with("```\n"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reset_memory_workspace_baseline_removes_generated_diff() {
|
||||
let home = TempDir::new().expect("tempdir");
|
||||
let root = home.path().join("memories");
|
||||
prepare_memory_workspace(&root)
|
||||
.await
|
||||
.expect("prepare memory workspace");
|
||||
fs::write(root.join("MEMORY.md"), "memory").expect("write memory");
|
||||
write_workspace_diff(
|
||||
&root,
|
||||
&GitBaselineDiff {
|
||||
changes: vec![GitBaselineChange {
|
||||
status: GitBaselineChangeStatus::Added,
|
||||
path: "MEMORY.md".to_string(),
|
||||
}],
|
||||
unified_diff: "+memory\n".to_string(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write workspace diff");
|
||||
|
||||
reset_memory_workspace_baseline(&root)
|
||||
.await
|
||||
.expect("reset baseline");
|
||||
|
||||
assert!(!root.join(WORKSPACE_DIFF_FILENAME).exists());
|
||||
let diff = memory_workspace_diff(&root)
|
||||
.await
|
||||
.expect("load workspace diff");
|
||||
assert_eq!(diff.changes, Vec::new());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepare_memory_workspace_recovers_unusable_git_dir() {
|
||||
let home = TempDir::new().expect("tempdir");
|
||||
let root = home.path().join("memories");
|
||||
fs::create_dir_all(root.join(".git")).expect("create unusable git dir");
|
||||
fs::write(root.join("MEMORY.md"), "memory").expect("write memory");
|
||||
|
||||
prepare_memory_workspace(&root)
|
||||
.await
|
||||
.expect("prepare memory workspace");
|
||||
|
||||
let diff = memory_workspace_diff(&root)
|
||||
.await
|
||||
.expect("load workspace diff");
|
||||
assert_eq!(diff.changes, Vec::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn previous_char_boundary_handles_multibyte_text() {
|
||||
let text = "aé";
|
||||
assert_eq!(previous_char_boundary(text, /*max_bytes*/ 2), 1);
|
||||
}
|
||||
@@ -1,38 +1,17 @@
|
||||
use crate::tools::context::ToolInvocation;
|
||||
use crate::tools::context::ToolPayload;
|
||||
use crate::tools::handlers::unified_exec::ExecCommandArgs;
|
||||
use codex_memories_read::usage::MEMORIES_USAGE_METRIC;
|
||||
use codex_memories_read::usage::memories_usage_kinds_from_command;
|
||||
use codex_protocol::models::ShellCommandToolCallParams;
|
||||
use codex_protocol::models::ShellToolCallParams;
|
||||
use codex_protocol::parse_command::ParsedCommand;
|
||||
use codex_shell_command::is_safe_command::is_known_safe_command;
|
||||
use codex_shell_command::parse_command::parse_command;
|
||||
use std::path::PathBuf;
|
||||
|
||||
const MEMORIES_USAGE_METRIC: &str = "codex.memories.usage";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
enum MemoriesUsageKind {
|
||||
MemoryMd,
|
||||
MemorySummary,
|
||||
RawMemories,
|
||||
RolloutSummaries,
|
||||
Skills,
|
||||
}
|
||||
|
||||
impl MemoriesUsageKind {
|
||||
fn as_tag(self) -> &'static str {
|
||||
match self {
|
||||
Self::MemoryMd => "memory_md",
|
||||
Self::MemorySummary => "memory_summary",
|
||||
Self::RawMemories => "raw_memories",
|
||||
Self::RolloutSummaries => "rollout_summaries",
|
||||
Self::Skills => "skills",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn emit_metric_for_tool_read(invocation: &ToolInvocation, success: bool) {
|
||||
let kinds = memories_usage_kinds_from_invocation(invocation).await;
|
||||
let Some((command, _)) = shell_command_for_invocation(invocation) else {
|
||||
return;
|
||||
};
|
||||
let kinds = memories_usage_kinds_from_command(&command);
|
||||
if kinds.is_empty() {
|
||||
return;
|
||||
}
|
||||
@@ -52,27 +31,6 @@ pub(crate) async fn emit_metric_for_tool_read(invocation: &ToolInvocation, succe
|
||||
}
|
||||
}
|
||||
|
||||
async fn memories_usage_kinds_from_invocation(
|
||||
invocation: &ToolInvocation,
|
||||
) -> Vec<MemoriesUsageKind> {
|
||||
let Some((command, _)) = shell_command_for_invocation(invocation) else {
|
||||
return Vec::new();
|
||||
};
|
||||
if !is_known_safe_command(&command) {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let parsed_commands = parse_command(&command);
|
||||
parsed_commands
|
||||
.into_iter()
|
||||
.filter_map(|command| match command {
|
||||
ParsedCommand::Read { path, .. } => get_memory_kind(path.display().to_string()),
|
||||
ParsedCommand::Search { path, .. } => path.and_then(get_memory_kind),
|
||||
ParsedCommand::ListFiles { .. } | ParsedCommand::Unknown { .. } => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn shell_command_for_invocation(invocation: &ToolInvocation) -> Option<(Vec<String>, PathBuf)> {
|
||||
let ToolPayload::Function { arguments } = &invocation.payload else {
|
||||
return None;
|
||||
@@ -129,19 +87,3 @@ fn shell_command_for_invocation(invocation: &ToolInvocation) -> Option<(Vec<Stri
|
||||
(Some(_), _) | (None, _) => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn get_memory_kind(path: String) -> Option<MemoriesUsageKind> {
|
||||
if path.contains("memories/MEMORY.md") {
|
||||
Some(MemoriesUsageKind::MemoryMd)
|
||||
} else if path.contains("memories/memory_summary.md") {
|
||||
Some(MemoriesUsageKind::MemorySummary)
|
||||
} else if path.contains("memories/raw_memories.md") {
|
||||
Some(MemoriesUsageKind::RawMemories)
|
||||
} else if path.contains("memories/rollout_summaries/") {
|
||||
Some(MemoriesUsageKind::RolloutSummaries)
|
||||
} else if path.contains("memories/skills/") {
|
||||
Some(MemoriesUsageKind::Skills)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -681,7 +681,7 @@ pub async fn drop_memories(sess: &Arc<Session>, config: &Arc<Config>, sub_id: St
|
||||
errors.push("state db unavailable; memory rows were not cleared".to_string());
|
||||
}
|
||||
|
||||
if let Err(err) = crate::memories::clear_memory_roots_contents(&config.codex_home).await {
|
||||
if let Err(err) = codex_memories_write::clear_memory_roots_contents(&config.codex_home).await {
|
||||
errors.push(format!(
|
||||
"failed clearing memory directories under {}: {err}",
|
||||
config.codex_home.display()
|
||||
@@ -689,7 +689,7 @@ pub async fn drop_memories(sess: &Arc<Session>, config: &Arc<Config>, sub_id: St
|
||||
}
|
||||
|
||||
if errors.is_empty() {
|
||||
let memory_root = crate::memories::memory_root(&config.codex_home);
|
||||
let memory_root = codex_memories_write::memory_root(&config.codex_home);
|
||||
sess.send_event_raw(Event {
|
||||
id: sub_id,
|
||||
msg: EventMsg::Warning(WarningEvent {
|
||||
|
||||
@@ -3323,7 +3323,7 @@ fn errors_to_info(errors: &[SkillError]) -> Vec<SkillErrorInfo> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
use crate::memories::prompts::build_memory_tool_developer_instructions;
|
||||
use codex_memories_read::build_memory_tool_developer_instructions;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests;
|
||||
|
||||
@@ -11,13 +11,13 @@ use tokio_util::sync::CancellationToken;
|
||||
use crate::context::ContextualUserFragment;
|
||||
use crate::context::ImageGenerationInstructions;
|
||||
use crate::function_tool::FunctionCallError;
|
||||
use crate::memories::citations::parse_memory_citation;
|
||||
use crate::memories::citations::thread_ids_from_memory_citation;
|
||||
use crate::parse_turn_item;
|
||||
use crate::session::session::Session;
|
||||
use crate::session::turn_context::TurnContext;
|
||||
use crate::tools::parallel::ToolCallRuntime;
|
||||
use crate::tools::router::ToolRouter;
|
||||
use codex_memories_read::citations::parse_memory_citation;
|
||||
use codex_memories_read::citations::thread_ids_from_memory_citation;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::goals::GoalRuntimeEvent;
|
||||
use crate::hook_runtime::record_additional_contexts;
|
||||
use crate::hook_runtime::run_post_tool_use_hooks;
|
||||
use crate::hook_runtime::run_pre_tool_use_hooks;
|
||||
use crate::memories::usage::emit_metric_for_tool_read;
|
||||
use crate::memory_usage::emit_metric_for_tool_read;
|
||||
use crate::sandbox_tags::permission_profile_policy_tag;
|
||||
use crate::sandbox_tags::permission_profile_sandbox_tag;
|
||||
use crate::session::turn_context::TurnContext;
|
||||
|
||||
Reference in New Issue
Block a user