feat: use git-backed workspace diffs for memory consolidation (#18982)

## Why

This PR make the `morpheus` agent (memory phase 2) use a git diff to
start it's consolidation. The workflow is the following:
1. The agent acquire a lock
2. If `.codex/memories` does not exist or is not a git root, initialize
everything (and make a first empty commit)
3. Update `raw_memories.md` and `rollout_summaries/` as before.
Basically we select max N phase 1 memories based on a given policy
4. We use git (`gix`) to get a diff between the current state of
`.codex/memories` and the last commit.
5. Dump the diff in `phase2_workspace_diff.md`
6. Spawn `morpheus` and point it to `phase2_workspace_diff.md`
7. Wait for `morpheus` to be done
8. Re-create a new `.git` and make one single commit on it. We do this
because we don't want to preserve history through `.git` and this is
cheap anyway
9. We release the lock
On top of this, we keep the retry policies etc etc

The goals of this new workflow are:
* Better support of any memory extensions such as `chronicle`
* Allow the user to manually edit memories and this will be considered
by the phase 2 agent
 
As a follow-up we will need to add support for user's edition while
`morpheus` is running

## What Changed

- Added memory workspace helpers that prepare the git baseline, compute
the diff, write `phase2_workspace_diff.md`, and reset the baseline after
successful consolidation.
- Updated Phase 2 to sync current inputs into `raw_memories.md` and
`rollout_summaries/`, prune old extension resources, skip clean
workspaces, and run the consolidation subagent only when the workspace
has changes.
- Tightened Phase 2 job ownership around long-running consolidation with
heartbeats and an ownership check before resetting the baseline.
- Simplified the prompt and state APIs so DB watermarks are bookkeeping,
while workspace dirtiness decides whether consolidation work exists.
- Updated the memory pipeline README and tests for workspace diffs,
extension-resource cleanup, pollution-driven forgetting, selection
ranking, and baseline persistence.

## Verification

- Added/updated coverage in `core/src/memories/tests.rs`,
`core/src/memories/workspace_tests.rs`, `state/src/runtime/memories.rs`,
and `core/tests/suite/memories.rs`.

---------

Co-authored-by: Codex <noreply@openai.com>

codex-rs/git-utils/src/baseline.rs

@@ -12,6 +12,8 @@ use std::path::Path;
 use std::path::PathBuf;
 use tokio::task;
 
+use crate::operations::run_git_for_status;
+
 const BASELINE_COMMIT_MESSAGE: &str =
     "Initialize Codex git baseline\n\nCo-authored-by: Codex <noreply@openai.com>";
 
@@ -65,18 +67,40 @@ struct GitBaselineFileEntry {
 /// This is intentionally destructive for `root/.git`. It is meant for internal directories where
 /// git is used only as a baseline/diff implementation detail, not for user repositories.
 pub async fn reset_git_repository(root: &Path) -> anyhow::Result<()> {
+    let root = root.to_path_buf();
+    task::spawn_blocking(move || reset_git_repository_sync(&root)).await?
+}
+
+/// Ensures `root` has a usable git baseline repository.
+///
+/// Existing usable `.git/` metadata is preserved. Missing or unusable metadata is replaced with a
+/// fresh one-commit baseline.
+pub async fn ensure_git_baseline_repository(root: &Path) -> anyhow::Result<()> {
     let root = root.to_path_buf();
     task::spawn_blocking(move || {
         fs::create_dir_all(&root)
             .with_context(|| format!("create git baseline root {}", root.display()))?;
-        remove_git_metadata(&root)?;
-        let repo = gix::init(&root).with_context(|| format!("init git repo {}", root.display()))?;
-        commit_current_tree(&repo, BASELINE_COMMIT_MESSAGE)?;
-        anyhow::Ok(())
+        if root.join(".git").is_dir()
+            && let Ok(repo) = gix::open(&root)
+            && head_file_entries(&repo).is_ok()
+        {
+            return Ok(());
+        }
+        reset_git_repository_sync(&root)
     })
     .await?
 }
 
+fn reset_git_repository_sync(root: &Path) -> anyhow::Result<()> {
+    fs::create_dir_all(root)
+        .with_context(|| format!("create git baseline root {}", root.display()))?;
+    remove_git_metadata(root)?;
+    let repo = gix::init(root).with_context(|| format!("init git repo {}", root.display()))?;
+    commit_current_tree(&repo, BASELINE_COMMIT_MESSAGE)?;
+    write_index_from_head(root)?;
+    Ok(())
+}
+
 /// Returns the diff between the latest baseline reset and the current directory contents.
 pub async fn diff_since_latest_init(root: &Path) -> anyhow::Result<GitBaselineDiff> {
     let root = root.to_path_buf();
@@ -130,6 +154,11 @@ fn commit_current_tree(repo: &gix::Repository, message: &str) -> anyhow::Result<
     Ok(())
 }
 
+fn write_index_from_head(root: &Path) -> anyhow::Result<()> {
+    run_git_for_status(root, ["read-tree", "--reset", "HEAD"], /*env*/ None)
+        .context("write git baseline index from HEAD")
+}
+
 fn codex_signature() -> gix::actor::Signature {
     gix::actor::Signature {
         name: "Codex".into(),
@@ -501,8 +530,24 @@ mod tests {
     use super::*;
     use pretty_assertions::assert_eq;
     use std::fs;
+    use std::process::Command;
     use tempfile::TempDir;
 
+    fn git_stdout(root: &Path, args: &[&str]) -> String {
+        let output = Command::new("git")
+            .current_dir(root)
+            .args(args)
+            .output()
+            .expect("run git command");
+        assert!(
+            output.status.success(),
+            "git command failed: {args:?}\nstdout:\n{}\nstderr:\n{}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
+        String::from_utf8_lossy(&output.stdout).to_string()
+    }
+
     #[tokio::test]
     async fn reset_creates_fresh_baseline() {
         let home = TempDir::new().expect("tempdir");
@@ -513,9 +558,30 @@ mod tests {
         reset_git_repository(&root).await.expect("reset repo");
 
         assert!(root.join(".git").is_dir());
+        assert!(root.join(".git/index").is_file());
         let diff = diff_since_latest_init(&root).await.expect("diff");
         assert!(!diff.has_changes());
         assert_eq!(diff.unified_diff, "");
+        assert_eq!(git_stdout(&root, &["status", "--porcelain"]), "");
+        assert_eq!(git_stdout(&root, &["ls-files"]), "MEMORY.md\n");
+    }
+
+    #[tokio::test]
+    async fn ensure_recovers_from_unborn_repository() {
+        let home = TempDir::new().expect("tempdir");
+        let root = home.path().join("repo");
+        fs::create_dir_all(&root).expect("create root");
+        fs::write(root.join("MEMORY.md"), "memory").expect("write memory");
+        gix::init(&root).expect("init git repo without baseline commit");
+
+        ensure_git_baseline_repository(&root)
+            .await
+            .expect("ensure repo");
+
+        let diff = diff_since_latest_init(&root).await.expect("diff");
+        assert!(!diff.has_changes());
+        assert_eq!(git_stdout(&root, &["status", "--porcelain"]), "");
+        assert_eq!(git_stdout(&root, &["ls-files"]), "MEMORY.md\n");
     }
 
     #[tokio::test]

codex-rs/git-utils/src/lib.rs

@@ -17,6 +17,7 @@ pub use baseline::GitBaselineChange;
 pub use baseline::GitBaselineChangeStatus;
 pub use baseline::GitBaselineDiff;
 pub use baseline::diff_since_latest_init;
+pub use baseline::ensure_git_baseline_repository;
 pub use baseline::reset_git_repository;
 pub use branch::merge_base_with_head;
 pub use codex_protocol::models::GhostCommit;