agent/codex
2
0
Fork 0
mirror of https://github.com/pchuan98/codex.git synced 2026-07-01 00:31:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

[codex] Migrate thread turns list to thread store (#19280)

Browse Source 
- migrate `thread/turns/list` to ThreadStore. Uses ThreadStore for most
data now but merges in the in-memory state from thread manager
- keep v2 `thread/list` pathless-store friendly by converting
`StoredThread` directly to API `Thread`
- add regression coverage for pathless store history/listing
This commit is contained in:
Tom
2026-04-30 14:16:42 -07:00
committed by GitHub
Unverified
parent 9121132c8f
commit 127be0612c
12 changed files with 713 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/app-server/src/codex_message_processor.rs
+223 -163
View File
@@ -268,7 +268,6 @@ use codex_core::exec::ExecParams;
use codex_core::exec_env::create_env;
use codex_core::find_archived_thread_path_by_id_str;
use codex_core::find_thread_name_by_id;
use codex_core::find_thread_names_by_ids;
use codex_core::find_thread_path_by_id_str;
use codex_core::path_utils;
use codex_core::plugins::PluginInstallError as CorePluginInstallError;
@@ -377,7 +376,6 @@ use codex_state::ThreadMetadata;
use codex_state::ThreadMetadataBuilder;
use codex_state::log_db::LogDbLayer;
use codex_thread_store::ArchiveThreadParams as StoreArchiveThreadParams;
#[cfg(debug_assertions)]
use codex_thread_store::InMemoryThreadStore;
use codex_thread_store::ListThreadsParams as StoreListThreadsParams;
use codex_thread_store::LocalThreadStore;
@@ -699,7 +697,6 @@ fn thread_store_from_config(config: &Config) -> Arc<dyn ThreadStore> {
match &config.experimental_thread_store {
ThreadStoreConfig::Local => Arc::new(configured_local_thread_store(config)),
ThreadStoreConfig::Remote { endpoint } => Arc::new(RemoteThreadStore::new(endpoint)),
#[cfg(debug_assertions)]
ThreadStoreConfig::InMemory { id } => InMemoryThreadStore::for_id(id),
}
}
@@ -3760,7 +3757,7 @@ impl CodexMessageProcessor {
ThreadSortKey::UpdatedAt => StoreThreadSortKey::UpdatedAt,
};
let sort_direction = sort_direction.unwrap_or(SortDirection::Desc);
let (summaries, next_cursor) = self
let (stored_threads, next_cursor) = self
.list_threads_common(
requested_page_size,
cursor,
@@ -3776,24 +3773,23 @@ impl CodexMessageProcessor {
},
)
.await?;
let backwards_cursor = summaries.first().and_then(|summary| {
thread_backwards_cursor_for_sort_key(summary, store_sort_key, sort_direction)
let backwards_cursor = stored_threads.first().and_then(|thread| {
thread_backwards_cursor_for_sort_key(thread, store_sort_key, sort_direction)
});
let mut threads = Vec::with_capacity(summaries.len());
let mut thread_ids = HashSet::with_capacity(summaries.len());
let mut status_ids = Vec::with_capacity(summaries.len());
let mut threads = Vec::with_capacity(stored_threads.len());
let mut status_ids = Vec::with_capacity(stored_threads.len());
let fallback_provider = self.config.model_provider_id.clone();
for summary in summaries {
let conversation_id = summary.conversation_id;
thread_ids.insert(conversation_id);
let thread = summary_to_thread(summary, &self.config.cwd);
for stored_thread in stored_threads {
let (thread, _) = thread_from_stored_thread(
stored_thread,
fallback_provider.as_str(),
&self.config.cwd,
);
status_ids.push(thread.id.clone());
threads.push((conversation_id, thread));
threads.push(thread);
}
let names = thread_titles_by_ids(&self.config, &thread_ids).await;
let statuses = self
.thread_watch_manager
.loaded_statuses_for_threads(status_ids)
@@ -3801,10 +3797,7 @@ impl CodexMessageProcessor {
let data: Vec<_> = threads
.into_iter()
.map(|(conversation_id, mut thread)| {
if let Some(title) = names.get(&conversation_id).cloned() {
set_thread_name_from_title(&mut thread, title);
}
.map(|mut thread| {
if let Some(status) = statuses.get(&thread.id) {
thread.status = status.clone();
}
@@ -3904,7 +3897,7 @@ impl CodexMessageProcessor {
thread_id: ThreadId,
include_turns: bool,
) -> Result<Thread, ThreadReadViewError> {
let loaded_thread = self.load_live_thread_for_read(thread_id).await;
let loaded_thread = self.thread_manager.get_thread(thread_id).await.ok();
let mut thread = if let Some(thread) = self
.load_persisted_thread_for_read(thread_id, include_turns)
.await?
@@ -3940,10 +3933,6 @@ impl CodexMessageProcessor {
Ok(thread)
}
async fn load_live_thread_for_read(&self, thread_id: ThreadId) -> Option<Arc<CodexThread>> {
self.thread_manager.get_thread(thread_id).await.ok()
}
async fn load_persisted_thread_for_read(
&self,
thread_id: ThreadId,
@@ -4071,87 +4060,37 @@ impl CodexMessageProcessor {
let thread_uuid = ThreadId::from_string(&thread_id)
.map_err(|err| invalid_request(format!("invalid thread id: {err}")))?;
let state_db_ctx = get_state_db(&self.config).await;
let mut rollout_path = self
.resolve_rollout_path(thread_uuid, state_db_ctx.as_ref())
.await;
if rollout_path.is_none() {
rollout_path =
match find_thread_path_by_id_str(&self.config.codex_home, &thread_uuid.to_string())
.await
{
Ok(Some(path)) => Some(path),
Ok(None) => match find_archived_thread_path_by_id_str(
&self.config.codex_home,
&thread_uuid.to_string(),
)
.await
{
Ok(path) => path,
Err(err) => {
return Err(invalid_request(format!(
"failed to locate archived thread id {thread_uuid}: {err}"
)));
}
},
Err(err) => {
return Err(invalid_request(format!(
"failed to locate thread id {thread_uuid}: {err}"
)));
}
};
}
if rollout_path.is_none() {
match self.thread_manager.get_thread(thread_uuid).await {
Ok(thread) => {
rollout_path = thread.rollout_path();
if rollout_path.is_none() {
return Err(invalid_request(
"ephemeral threads do not support thread/turns/list",
));
}
}
Err(_) => return Err(invalid_request(format!("thread not loaded: {thread_uuid}"))),
}
}
let Some(rollout_path) = rollout_path.as_ref() else {
return Err(internal_error(format!(
"failed to locate rollout for thread {thread_uuid}"
)));
};
let items = match read_rollout_items_from_rollout(rollout_path).await {
Ok(items) => items,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
return Err(invalid_request(format!(
"thread {thread_uuid} is not materialized yet; thread/turns/list is unavailable before first user message"
)));
}
Err(err) => {
return Err(internal_error(format!(
"failed to load rollout `{}` for thread {thread_uuid}: {err}",
rollout_path.display()
)));
}
};
let items = self
.load_thread_turns_list_history(thread_uuid)
.await
.map_err(thread_read_view_error)?;
// This API optimizes network transfer by letting clients page through a
// thread's turns incrementally, but it still replays the entire rollout on
// every request. Rollback and compaction events can change earlier turns, so
// the server has to rebuild the full turn list until turn metadata is indexed
// separately.
let has_live_in_progress_turn = match self.thread_manager.get_thread(thread_uuid).await {
Ok(thread) => matches!(thread.agent_status().await, AgentStatus::Running),
Err(_) => false,
let loaded_thread = self.thread_manager.get_thread(thread_uuid).await.ok();
let has_live_running_thread = match loaded_thread.as_ref() {
Some(thread) => matches!(thread.agent_status().await, AgentStatus::Running),
None => false,
};
let turns = reconstruct_thread_turns_from_rollout_items(
let active_turn = if loaded_thread.is_some() {
// Persisted history may not yet include the currently running turn. The
// app-server listener has already projected live turn events into ThreadState,
// so merge that in-memory snapshot before paginating.
let thread_state = self.thread_state_manager.thread_state(thread_uuid).await;
let state = thread_state.lock().await;
state.active_turn_snapshot()
} else {
None
};
let turns = reconstruct_thread_turns_for_turns_list(
&items,
self.thread_watch_manager
.loaded_status_for_thread(&thread_uuid.to_string())
.await,
has_live_in_progress_turn,
has_live_running_thread,
active_turn,
);
let page = paginate_thread_turns(
turns,
@@ -4166,6 +4105,63 @@ impl CodexMessageProcessor {
})
}
async fn load_thread_turns_list_history(
&self,
thread_id: ThreadId,
) -> Result<Vec<RolloutItem>, ThreadReadViewError> {
match self
.thread_store
.read_thread(StoreReadThreadParams {
thread_id,
include_archived: true,
include_history: true,
})
.await
{
Ok(stored_thread) => {
let history = stored_thread.history.ok_or_else(|| {
ThreadReadViewError::Internal(format!(
"thread store did not return history for thread {thread_id}"
))
})?;
return Ok(history.items);
}
Err(ThreadStoreError::InvalidRequest { message })
if message == format!("no rollout found for thread id {thread_id}") => {}
Err(ThreadStoreError::ThreadNotFound {
thread_id: missing_thread_id,
}) if missing_thread_id == thread_id => {}
Err(ThreadStoreError::InvalidRequest { message }) => {
return Err(ThreadReadViewError::InvalidRequest(message));
}
Err(err) => {
return Err(ThreadReadViewError::Internal(format!(
"failed to read thread: {err}"
)));
}
}
let thread = self
.thread_manager
.get_thread(thread_id)
.await
.map_err(|_| {
ThreadReadViewError::InvalidRequest(format!("thread not loaded: {thread_id}"))
})?;
let config_snapshot = thread.config_snapshot().await;
if config_snapshot.ephemeral {
return Err(ThreadReadViewError::InvalidRequest(
"ephemeral threads do not support thread/turns/list".to_string(),
));
}
thread
.load_history(/*include_archived*/ true)
.await
.map(|history| history.items)
.map_err(|err| thread_turns_list_history_load_error(thread_id, err))
}
pub(crate) fn thread_created_receiver(&self) -> broadcast::Receiver<ThreadId> {
self.thread_manager.subscribe_thread_created()
}
@@ -4722,12 +4718,11 @@ impl CodexMessageProcessor {
let (mut thread, history) =
thread_from_stored_thread(stored_thread, fallback_provider, &self.config.cwd);
if include_turns && let Some(history) = history {
populate_thread_turns(
populate_thread_turns_from_history(
&mut thread,
ThreadTurnSource::HistoryItems(&history.items),
&history.items,
/*active_turn*/ None,
)
.await?;
)?;
}
Ok(thread)
}
@@ -4761,8 +4756,33 @@ impl CodexMessageProcessor {
let thread = match thread_history {
InitialHistory::Resumed(resumed) => {
let fallback_provider = config_snapshot.model_provider_id.as_str();
if let Some(mut stored_thread) = resume_source_thread {
stored_thread.history = None;
if let Some(stored_thread) = resume_source_thread {
let stored_thread =
if let Some(rollout_path) = stored_thread.rollout_path.clone() {
self.thread_store
.read_thread_by_rollout_path(StoreReadThreadByRolloutPathParams {
rollout_path,
include_archived: true,
include_history: false,
})
.await
.unwrap_or(StoredThread {
history: None,
..stored_thread
})
} else {
self.thread_store
.read_thread(StoreReadThreadParams {
thread_id: stored_thread.thread_id,
include_archived: true,
include_history: false,
})
.await
.unwrap_or(StoredThread {
history: None,
..stored_thread
})
};
Ok(thread_from_stored_thread(
stored_thread,
fallback_provider,
@@ -4809,12 +4829,11 @@ impl CodexMessageProcessor {
thread.path = Some(rollout_path.to_path_buf());
if include_turns {
let history_items = thread_history.get_rollout_items();
populate_thread_turns(
populate_thread_turns_from_history(
&mut thread,
ThreadTurnSource::HistoryItems(&history_items),
&history_items,
/*active_turn*/ None,
)
.await?;
)?;
}
self.attach_thread_name(thread_id, &mut thread).await;
Ok(thread)
@@ -4991,12 +5010,11 @@ impl CodexMessageProcessor {
thread.preview = preview_from_rollout_items(&history_items);
thread.forked_from_id = Some(source_thread_id.to_string());
if include_turns {
populate_thread_turns(
populate_thread_turns_from_history(
&mut thread,
ThreadTurnSource::HistoryItems(&history_items),
&history_items,
/*active_turn*/ None,
)
.await
.map_err(internal_error)?;
}
thread
@@ -5150,7 +5168,7 @@ impl CodexMessageProcessor {
sort_key: StoreThreadSortKey,
sort_direction: SortDirection,
filters: ThreadListFilters,
) -> Result<(Vec<ConversationSummary>, Option<String>), JSONRPCErrorError> {
) -> Result<(Vec<StoredThread>, Option<String>), JSONRPCErrorError> {
let ThreadListFilters {
model_providers,
source_kinds,
@@ -5175,7 +5193,6 @@ impl CodexMessageProcessor {
}
None => Some(vec![self.config.model_provider_id.clone()]),
};
let fallback_provider = self.config.model_provider_id.clone();
let (allowed_sources_vec, source_kind_filter) = compute_source_filters(source_kinds);
let allowed_sources = allowed_sources_vec.as_slice();
let store_sort_direction = match sort_direction {
@@ -5204,20 +5221,21 @@ impl CodexMessageProcessor {
let mut filtered = Vec::with_capacity(page.items.len());
for it in page.items {
let Some(summary) = summary_from_stored_thread(it, fallback_provider.as_str())
else {
continue;
};
let source = with_thread_spawn_agent_metadata(
it.source.clone(),
it.agent_nickname.clone(),
it.agent_role.clone(),
);
if source_kind_filter
.as_ref()
.is_none_or(|filter| source_kind_matches(&summary.source, filter))
.is_none_or(|filter| source_kind_matches(&source, filter))
&& cwd_filters.as_ref().is_none_or(|expected_cwds| {
expected_cwds.iter().any(|expected_cwd| {
path_utils::paths_match_after_normalization(&summary.cwd, expected_cwd)
path_utils::paths_match_after_normalization(&it.cwd, expected_cwd)
})
})
{
filtered.push(summary);
filtered.push(it);
if filtered.len() >= remaining {
break;
}
@@ -8317,12 +8335,11 @@ async fn handle_pending_thread_resume_request(
let connection_id = request_id.connection_id;
let mut thread = pending.thread_summary;
if pending.include_turns
&& let Err(message) = populate_thread_turns(
&& let Err(message) = populate_thread_turns_from_history(
&mut thread,
ThreadTurnSource::HistoryItems(&pending.history_items),
&pending.history_items,
active_turn.as_ref(),
)
.await
{
outgoing
.send_error(request_id, internal_error(message))
@@ -8482,18 +8499,12 @@ async fn send_thread_goal_snapshot_notification(
}
}
enum ThreadTurnSource<'a> {
HistoryItems(&'a [RolloutItem]),
}
async fn populate_thread_turns(
fn populate_thread_turns_from_history(
thread: &mut Thread,
turn_source: ThreadTurnSource<'_>,
items: &[RolloutItem],
active_turn: Option<&Turn>,
) -> std::result::Result<(), String> {
let mut turns = match turn_source {
ThreadTurnSource::HistoryItems(items) => build_turns_from_rollout_items(items),
};
let mut turns = build_turns_from_rollout_items(items);
if let Some(active_turn) = active_turn {
merge_turn_history_with_active_turn(&mut turns, active_turn.clone());
}
@@ -8959,31 +8970,6 @@ async fn title_from_state_db(config: &Config, thread_id: ThreadId) -> Option<Str
.flatten()
}
async fn thread_titles_by_ids(
config: &Config,
thread_ids: &HashSet<ThreadId>,
) -> HashMap<ThreadId, String> {
let mut names = HashMap::with_capacity(thread_ids.len());
if let Some(state_db_ctx) = open_state_db_for_direct_thread_lookup(config).await {
for &thread_id in thread_ids {
let Ok(Some(metadata)) = state_db_ctx.get_thread(thread_id).await else {
continue;
};
if let Some(title) = distinct_title(&metadata) {
names.insert(thread_id, title);
}
}
}
if names.len() < thread_ids.len()
&& let Ok(legacy_names) = find_thread_names_by_ids(&config.codex_home, thread_ids).await
{
for (thread_id, title) in legacy_names {
names.entry(thread_id).or_insert(title);
}
}
names
}
async fn open_state_db_for_direct_thread_lookup(config: &Config) -> Option<StateDbHandle> {
StateRuntime::init(config.sqlite_home.clone(), config.model_provider_id.clone())
.await
@@ -9067,6 +9053,27 @@ fn thread_store_resume_read_error(err: ThreadStoreError) -> JSONRPCErrorError {
}
}
fn thread_turns_list_history_load_error(
thread_id: ThreadId,
err: ThreadStoreError,
) -> ThreadReadViewError {
match err {
ThreadStoreError::InvalidRequest { message }
if message.starts_with("failed to resolve rollout path `") =>
{
ThreadReadViewError::InvalidRequest(format!(
"thread {thread_id} is not materialized yet; thread/turns/list is unavailable before first user message"
))
}
ThreadStoreError::InvalidRequest { message } => {
ThreadReadViewError::InvalidRequest(message)
}
err => ThreadReadViewError::Internal(format!(
"failed to load thread history for thread {thread_id}: {err}"
)),
}
}
fn conversation_summary_thread_id_read_error(
conversation_id: ThreadId,
err: ThreadStoreError,
@@ -9726,18 +9733,14 @@ pub(crate) fn summary_to_thread(
}
fn thread_backwards_cursor_for_sort_key(
summary: &ConversationSummary,
thread: &StoredThread,
sort_key: StoreThreadSortKey,
sort_direction: SortDirection,
) -> Option<String> {
let timestamp = match sort_key {
StoreThreadSortKey::CreatedAt => summary.timestamp.as_deref(),
StoreThreadSortKey::UpdatedAt => summary
.updated_at
.as_deref()
.or(summary.timestamp.as_deref()),
StoreThreadSortKey::CreatedAt => thread.created_at,
StoreThreadSortKey::UpdatedAt => thread.updated_at,
};
let timestamp = parse_datetime(timestamp)?;
// The state DB stores unique millisecond timestamps. Offset the reverse cursor by one
// millisecond so the opposite-direction query includes the page anchor.
let timestamp = match sort_direction {
@@ -9874,6 +9877,27 @@ fn reconstruct_thread_turns_from_rollout_items(
turns
}
fn reconstruct_thread_turns_for_turns_list(
items: &[RolloutItem],
loaded_status: ThreadStatus,
has_live_running_thread: bool,
active_turn: Option<Turn>,
) -> Vec<Turn> {
let has_live_in_progress_turn = has_live_running_thread
|| active_turn
.as_ref()
.is_some_and(|turn| matches!(turn.status, TurnStatus::InProgress));
let mut turns = reconstruct_thread_turns_from_rollout_items(
items,
loaded_status,
has_live_in_progress_turn,
);
if let Some(active_turn) = active_turn {
merge_turn_history_with_active_turn(&mut turns, active_turn);
}
turns
}
fn normalize_thread_turns_status(
turns: &mut [Turn],
loaded_status: ThreadStatus,
@@ -10032,6 +10056,42 @@ mod tests {
assert!(err.contains("my_tool"), "unexpected error: {err}");
}
#[test]
fn thread_turns_list_merges_in_progress_active_turn_before_agent_status_running() {
let persisted_items = vec![RolloutItem::EventMsg(EventMsg::UserMessage(
codex_protocol::protocol::UserMessageEvent {
message: "persisted".to_string(),
images: None,
local_images: Vec::new(),
text_elements: Vec::new(),
},
))];
let active_turn = Turn {
id: "live-turn".to_string(),
items: vec![ThreadItem::UserMessage {
id: "live-user-message".to_string(),
content: vec![V2UserInput::Text {
text: "live".to_string(),
text_elements: Vec::new(),
}],
}],
error: None,
status: TurnStatus::InProgress,
started_at: None,
completed_at: None,
duration_ms: None,
};
let turns = reconstruct_thread_turns_for_turns_list(
&persisted_items,
ThreadStatus::Idle,
/*has_live_running_thread*/ false,
Some(active_turn.clone()),
);
assert_eq!(turns.last(), Some(&active_turn));
}
#[test]
fn validate_dynamic_tools_rejects_empty_namespace() {
let tools = vec![ApiDynamicToolSpec {
codex-rs/app-server/tests/suite/v2/remote_thread_store.rs
+27
View File
@@ -28,6 +28,8 @@ use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerNotification;
use codex_app_server_protocol::ThreadListParams;
use codex_app_server_protocol::ThreadListResponse;
use codex_app_server_protocol::ThreadStartParams;
use codex_app_server_protocol::ThreadStartResponse;
use codex_app_server_protocol::TurnStartParams;
@@ -136,10 +138,35 @@ async fn thread_start_with_non_local_thread_store_does_not_create_local_persiste
})
.await??;
let response = client
.request(ClientRequest::ThreadList {
request_id: RequestId::Integer(3),
params: ThreadListParams {
cursor: None,
limit: Some(10),
sort_key: None,
sort_direction: None,
model_providers: Some(Vec::new()),
source_kinds: None,
archived: None,
cwd: None,
use_state_db_only: false,
search_term: None,
},
})
.await?
.expect("thread/list should succeed");
let ThreadListResponse { data, .. } =
serde_json::from_value(response).expect("thread/list response should parse");
assert_eq!(data.len(), 1);
assert_eq!(data[0].id, thread.id);
assert_eq!(data[0].path, None);
client.shutdown().await?;
let calls = thread_store.calls().await;
assert_eq!(calls.create_thread, 1);
assert_eq!(calls.list_threads, 1);
assert!(
calls.append_items > 0,
"turn/start should append rollout items through the injected store"
codex-rs/app-server/tests/suite/v2/thread_read.rs
+298
View File
@@ -5,6 +5,12 @@ use app_test_support::create_mock_responses_server_repeating_assistant;
use app_test_support::rollout_path;
use app_test_support::test_absolute_path;
use app_test_support::to_response;
use codex_app_server::in_process;
use codex_app_server::in_process::InProcessStartArgs;
use codex_app_server_protocol::ClientInfo;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::InitializeCapabilities;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::JSONRPCError;
use codex_app_server_protocol::JSONRPCResponse;
use codex_app_server_protocol::RequestId;
@@ -31,17 +37,37 @@ use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::TurnStartResponse;
use codex_app_server_protocol::TurnStatus;
use codex_app_server_protocol::UserInput;
use codex_arg0::Arg0DispatchPaths;
use codex_config::CloudRequirementsLoader;
use codex_config::LoaderOverrides;
use codex_core::ARCHIVED_SESSIONS_SUBDIR;
use codex_core::config::ConfigBuilder;
use codex_exec_server::EnvironmentManager;
use codex_feedback::CodexFeedback;
use codex_protocol::models::BaseInstructions;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::SessionSource as ProtocolSessionSource;
use codex_protocol::protocol::UserMessageEvent;
use codex_protocol::user_input::ByteRange;
use codex_protocol::user_input::TextElement;
use codex_thread_store::AppendThreadItemsParams;
use codex_thread_store::CreateThreadParams;
use codex_thread_store::InMemoryThreadStore;
use codex_thread_store::ThreadEventPersistenceMode;
use codex_thread_store::ThreadMetadataPatch;
use codex_thread_store::ThreadStore;
use codex_thread_store::UpdateThreadMetadataParams;
use core_test_support::responses;
use pretty_assertions::assert_eq;
use serde_json::Value;
use serde_json::json;
use std::io::Write;
use std::path::Path;
use std::sync::Arc;
use tempfile::TempDir;
use tokio::time::timeout;
use uuid::Uuid;
#[cfg(windows)]
const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25);
@@ -246,6 +272,147 @@ async fn thread_turns_list_can_page_backward_and_forward() -> Result<()> {
Ok(())
}
#[tokio::test]
async fn thread_turns_list_reads_store_history_without_rollout_path() -> Result<()> {
let codex_home = TempDir::new()?;
let thread_id = codex_protocol::ThreadId::from_string("00000000-0000-4000-8000-000000000123")?;
let store_id = Uuid::new_v4().to_string();
create_config_toml_with_thread_store(codex_home.path(), &store_id)?;
let store = InMemoryThreadStore::for_id(store_id.clone());
let _in_memory_store = InMemoryThreadStoreId { store_id };
seed_pathless_store_thread(&store, thread_id).await?;
let loader_overrides = LoaderOverrides::without_managed_config_for_tests();
let config = ConfigBuilder::default()
.codex_home(codex_home.path().to_path_buf())
.fallback_cwd(Some(codex_home.path().to_path_buf()))
.loader_overrides(loader_overrides.clone())
.build()
.await?;
let client = in_process::start(InProcessStartArgs {
arg0_paths: Arg0DispatchPaths::default(),
config: Arc::new(config),
cli_overrides: Vec::new(),
loader_overrides,
cloud_requirements: CloudRequirementsLoader::default(),
thread_config_loader: Arc::new(codex_config::NoopThreadConfigLoader),
feedback: CodexFeedback::new(),
log_db: None,
environment_manager: Arc::new(EnvironmentManager::default_for_tests()),
config_warnings: Vec::new(),
session_source: SessionSource::Cli.into(),
enable_codex_api_key_env: false,
initialize: InitializeParams {
client_info: ClientInfo {
name: "codex-app-server-tests".to_string(),
title: None,
version: "0.1.0".to_string(),
},
capabilities: Some(InitializeCapabilities {
experimental_api: true,
..Default::default()
}),
},
channel_capacity: in_process::DEFAULT_IN_PROCESS_CHANNEL_CAPACITY,
})
.await?;
let result = client
.request(ClientRequest::ThreadTurnsList {
request_id: RequestId::Integer(1),
params: ThreadTurnsListParams {
thread_id: thread_id.to_string(),
cursor: None,
limit: Some(10),
sort_direction: Some(SortDirection::Asc),
},
})
.await?
.expect("thread/turns/list should succeed");
let ThreadTurnsListResponse { data, .. } = serde_json::from_value(result)?;
assert_eq!(turn_user_texts(&data), vec!["history from store"]);
client.shutdown().await?;
Ok(())
}
#[tokio::test]
async fn thread_list_includes_store_thread_without_rollout_path() -> Result<()> {
let codex_home = TempDir::new()?;
let thread_id = codex_protocol::ThreadId::from_string("00000000-0000-4000-8000-000000000124")?;
let store_id = Uuid::new_v4().to_string();
create_config_toml_with_thread_store(codex_home.path(), &store_id)?;
let store = InMemoryThreadStore::for_id(store_id.clone());
let _in_memory_store = InMemoryThreadStoreId { store_id };
seed_pathless_store_thread(&store, thread_id).await?;
let loader_overrides = LoaderOverrides::without_managed_config_for_tests();
let config = ConfigBuilder::default()
.codex_home(codex_home.path().to_path_buf())
.fallback_cwd(Some(codex_home.path().to_path_buf()))
.loader_overrides(loader_overrides.clone())
.build()
.await?;
let client = in_process::start(InProcessStartArgs {
arg0_paths: Arg0DispatchPaths::default(),
config: Arc::new(config),
cli_overrides: Vec::new(),
loader_overrides,
cloud_requirements: CloudRequirementsLoader::default(),
thread_config_loader: Arc::new(codex_config::NoopThreadConfigLoader),
feedback: CodexFeedback::new(),
log_db: None,
environment_manager: Arc::new(EnvironmentManager::default_for_tests()),
config_warnings: Vec::new(),
session_source: SessionSource::Cli.into(),
enable_codex_api_key_env: false,
initialize: InitializeParams {
client_info: ClientInfo {
name: "codex-app-server-tests".to_string(),
title: None,
version: "0.1.0".to_string(),
},
capabilities: Some(InitializeCapabilities {
experimental_api: true,
..Default::default()
}),
},
channel_capacity: in_process::DEFAULT_IN_PROCESS_CHANNEL_CAPACITY,
})
.await?;
let result = client
.request(ClientRequest::ThreadList {
request_id: RequestId::Integer(1),
params: ThreadListParams {
cursor: None,
limit: Some(10),
sort_key: None,
sort_direction: None,
model_providers: Some(Vec::new()),
source_kinds: None,
archived: None,
cwd: None,
use_state_db_only: false,
search_term: None,
},
})
.await?
.expect("thread/list should succeed");
let ThreadListResponse { data, .. } = serde_json::from_value(result)?;
assert_eq!(data.len(), 1);
let thread = &data[0];
assert_eq!(thread.id, thread_id.to_string());
assert_eq!(thread.path, None);
assert_eq!(thread.preview, "");
assert_eq!(thread.name.as_deref(), Some("named pathless thread"));
client.shutdown().await?;
Ok(())
}
#[tokio::test]
async fn thread_read_can_return_archived_threads_by_id() -> Result<()> {
let server = create_mock_responses_server_repeating_assistant("Done").await;
@@ -670,6 +837,59 @@ async fn thread_read_include_turns_rejects_unmaterialized_loaded_thread() -> Res
Ok(())
}
#[tokio::test]
async fn thread_turns_list_rejects_unmaterialized_loaded_thread() -> Result<()> {
let server = create_mock_responses_server_repeating_assistant("Done").await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let start_id = mcp
.send_thread_start_request(ThreadStartParams {
model: Some("mock-model".to_string()),
..Default::default()
})
.await?;
let start_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(start_id)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(start_resp)?;
let thread_path = thread.path.clone().expect("thread path");
assert!(
!thread_path.exists(),
"fresh thread rollout should not be materialized yet"
);
let read_id = mcp
.send_thread_turns_list_request(ThreadTurnsListParams {
thread_id: thread.id,
cursor: None,
limit: None,
sort_direction: None,
})
.await?;
let read_err: JSONRPCError = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_error_message(RequestId::Integer(read_id)),
)
.await??;
assert!(
read_err
.error
.message
.contains("thread/turns/list is unavailable before first user message"),
"unexpected error: {}",
read_err.error.message
);
Ok(())
}
#[tokio::test]
async fn thread_read_reports_system_error_idle_flag_after_failed_turn() -> Result<()> {
let server = responses::start_mock_server().await;
@@ -787,6 +1007,84 @@ fn turn_user_texts(turns: &[codex_app_server_protocol::Turn]) -> Vec<&str> {
.collect()
}
struct InMemoryThreadStoreId {
store_id: String,
}
impl Drop for InMemoryThreadStoreId {
fn drop(&mut self) {
InMemoryThreadStore::remove_id(&self.store_id);
}
}
async fn seed_pathless_store_thread(
store: &InMemoryThreadStore,
thread_id: codex_protocol::ThreadId,
) -> Result<()> {
store
.create_thread(CreateThreadParams {
thread_id,
forked_from_id: None,
source: ProtocolSessionSource::Cli,
base_instructions: BaseInstructions::default(),
dynamic_tools: Vec::new(),
event_persistence_mode: ThreadEventPersistenceMode::default(),
})
.await?;
store
.append_items(AppendThreadItemsParams {
thread_id,
items: store_history_items(),
})
.await?;
store
.update_thread_metadata(UpdateThreadMetadataParams {
thread_id,
patch: ThreadMetadataPatch {
name: Some("named pathless thread".to_string()),
..Default::default()
},
include_archived: true,
})
.await?;
Ok(())
}
fn store_history_items() -> Vec<RolloutItem> {
vec![RolloutItem::EventMsg(EventMsg::UserMessage(
UserMessageEvent {
message: "history from store".to_string(),
images: None,
local_images: Vec::new(),
text_elements: Vec::new(),
},
))]
}
fn create_config_toml_with_thread_store(codex_home: &Path, store_id: &str) -> std::io::Result<()> {
let config_toml = codex_home.join("config.toml");
std::fs::write(
config_toml,
format!(
r#"
model = "mock-model"
approval_policy = "never"
sandbox_mode = "read-only"
experimental_thread_store = {{ type = "in_memory", id = "{store_id}" }}
model_provider = "mock_provider"
[model_providers.mock_provider]
name = "Mock provider for test"
base_url = "http://127.0.0.1:1/v1"
wire_api = "responses"
request_max_retries = 0
stream_max_retries = 0
"#
),
)
}
// Helper to create a config.toml pointing at the mock model server.
fn create_config_toml(codex_home: &Path, server_uri: &str) -> std::io::Result<()> {
let config_toml = codex_home.join("config.toml");
codex-rs/config/src/config_toml.rs
-1
View File
@@ -427,7 +427,6 @@ pub enum ThreadStoreToml {
Remote {
endpoint: String,
},
#[cfg(debug_assertions)]
#[schemars(skip)]
InMemory {
id: String,
codex-rs/core/src/codex_thread.rs
+17
View File
@@ -31,6 +31,9 @@ use codex_protocol::protocol::ThreadMemoryMode;
use codex_protocol::protocol::TokenUsageInfo;
use codex_protocol::protocol::W3cTraceContext;
use codex_protocol::user_input::UserInput;
use codex_thread_store::StoredThreadHistory;
use codex_thread_store::ThreadStoreError;
use codex_thread_store::ThreadStoreResult;
use codex_utils_absolute_path::AbsolutePathBuf;
use rmcp::model::ReadResourceRequestParams;
use std::collections::HashMap;
@@ -382,6 +385,20 @@ impl CodexThread {
.await
}
pub async fn load_history(
&self,
include_archived: bool,
) -> ThreadStoreResult<StoredThreadHistory> {
let live_thread = self
.codex
.session
.live_thread_for_persistence("load history")
.map_err(|err| ThreadStoreError::Internal {
message: err.to_string(),
})?;
live_thread.load_history(include_archived).await
}
pub fn state_db(&self) -> Option<StateDbHandle> {
self.codex.state_db()
}
codex-rs/core/src/config/mod.rs
+1 -3
View File
@@ -375,8 +375,7 @@ pub enum ThreadStoreConfig {
Local,
/// Persist threads through the remote thread-store service.
Remote { endpoint: String },
/// Test-only in-memory thread store.
#[cfg(debug_assertions)]
/// In-memory thread store for test and debug configurations.
InMemory { id: String },
}
@@ -1586,7 +1585,6 @@ fn thread_store_config(
match thread_store {
Some(ThreadStoreToml::Local {}) => ThreadStoreConfig::Local,
Some(ThreadStoreToml::Remote { endpoint }) => ThreadStoreConfig::Remote { endpoint },
#[cfg(debug_assertions)]
Some(ThreadStoreToml::InMemory { id }) => ThreadStoreConfig::InMemory { id },
None => legacy_remote_endpoint.map_or(ThreadStoreConfig::Local, |endpoint| {
ThreadStoreConfig::Remote { endpoint }
codex-rs/core/src/thread_manager.rs
-2
View File
@@ -52,7 +52,6 @@ use codex_protocol::protocol::TurnEnvironmentSelection;
use codex_protocol::protocol::W3cTraceContext;
use codex_rollout::RolloutConfig;
use codex_state::DirectionalThreadSpawnEdgeStatus;
#[cfg(debug_assertions)]
use codex_thread_store::InMemoryThreadStore;
use codex_thread_store::LocalThreadStore;
use codex_thread_store::RemoteThreadStore;
@@ -268,7 +267,6 @@ pub fn thread_store_from_config(config: &Config) -> Arc<dyn ThreadStore> {
Arc::new(LocalThreadStore::new(RolloutConfig::from_view(config)))
}
ThreadStoreConfig::Remote { endpoint } => Arc::new(RemoteThreadStore::new(endpoint)),
#[cfg(debug_assertions)]
ThreadStoreConfig::InMemory { id } => InMemoryThreadStore::for_id(id),
}
}
codex-rs/thread-store/src/in_memory.rs
+2 -2
View File
@@ -61,9 +61,9 @@ pub struct InMemoryThreadStoreCalls {
pub unarchive_thread: usize,
}
/// Test-only in-memory [`ThreadStore`] implementation.
/// In-memory [`ThreadStore`] implementation for tests and debug configs.
///
/// Debug/test configs can select this store by id, letting tests exercise
/// Test and debug configs can select this store by id, letting tests exercise
/// config-driven non-local persistence without requiring the real remote gRPC
/// service.
#[derive(Default)]
codex-rs/thread-store/src/lib.rs
-3
View File
@@ -5,7 +5,6 @@
//! any other backing store.
mod error;
#[cfg(debug_assertions)]
mod in_memory;
mod live_thread;
mod local;
@@ -15,9 +14,7 @@ mod types;
pub use error::ThreadStoreError;
pub use error::ThreadStoreResult;
#[cfg(debug_assertions)]
pub use in_memory::InMemoryThreadStore;
#[cfg(debug_assertions)]
pub use in_memory::InMemoryThreadStoreCalls;
pub use live_thread::LiveThread;
pub use live_thread::LiveThreadInitGuard;
codex-rs/thread-store/src/local/helpers.rs
+17
View File
@@ -14,6 +14,7 @@ use codex_protocol::protocol::GitInfo;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionSource;
use codex_rollout::ThreadItem;
use codex_state::ThreadMetadata;
use crate::StoredThread;
use crate::ThreadStoreError;
@@ -133,6 +134,22 @@ pub(super) fn stored_thread_from_rollout_item(
})
}
pub(super) fn distinct_thread_metadata_title(metadata: &ThreadMetadata) -> Option<String> {
let title = metadata.title.trim();
if title.is_empty() || metadata.first_user_message.as_deref().map(str::trim) == Some(title) {
None
} else {
Some(title.to_string())
}
}
pub(super) fn set_thread_name_from_title(thread: &mut StoredThread, title: String) {
if title.trim().is_empty() || thread.preview.trim() == title.trim() {
return;
}
thread.name = Some(title);
}
fn parse_rfc3339(value: Option<&str>) -> Option<DateTime<Utc>> {
DateTime::parse_from_rfc3339(value?)
.ok()
codex-rs/thread-store/src/local/list_threads.rs
+37 -1
View File
@@ -1,8 +1,15 @@
use std::collections::HashMap;
use std::collections::HashSet;
use codex_protocol::ThreadId;
use codex_rollout::RolloutConfig;
use codex_rollout::RolloutRecorder;
use codex_rollout::find_thread_names_by_ids;
use codex_rollout::parse_cursor;
use super::LocalThreadStore;
use super::helpers::distinct_thread_metadata_title;
use super::helpers::set_thread_name_from_title;
use super::helpers::stored_thread_from_rollout_item;
use crate::ListThreadsParams;
use crate::SortDirection;
@@ -46,7 +53,7 @@ pub(super) async fn list_threads(
.as_ref()
.and_then(|cursor| serde_json::to_value(cursor).ok())
.and_then(|value| value.as_str().map(str::to_owned));
let items = page
let mut items = page
.items
.into_iter()
.filter_map(|item| {
@@ -58,6 +65,35 @@ pub(super) async fn list_threads(
})
.collect::<Vec<_>>();
let thread_ids = items
.iter()
.map(|thread| thread.thread_id)
.collect::<HashSet<_>>();
let mut names = HashMap::<ThreadId, String>::with_capacity(thread_ids.len());
if let Some(state_db_ctx) = store.state_db().await {
for &thread_id in &thread_ids {
let Ok(Some(metadata)) = state_db_ctx.get_thread(thread_id).await else {
continue;
};
if let Some(title) = distinct_thread_metadata_title(&metadata) {
names.insert(thread_id, title);
}
}
}
if names.len() < thread_ids.len()
&& let Ok(legacy_names) =
find_thread_names_by_ids(store.config.codex_home.as_path(), &thread_ids).await
{
for (thread_id, title) in legacy_names {
names.entry(thread_id).or_insert(title);
}
}
for thread in &mut items {
if let Some(title) = names.get(&thread.thread_id).cloned() {
set_thread_name_from_title(thread, title);
}
}
Ok(ThreadPage { items, next_cursor })
}
codex-rs/thread-store/src/local/read_thread.rs
+91 -17
View File
@@ -14,7 +14,9 @@ use codex_state::StateRuntime;
use codex_state::ThreadMetadata;
use super::LocalThreadStore;
use super::helpers::distinct_thread_metadata_title;
use super::helpers::git_info_from_parts;
use super::helpers::set_thread_name_from_title;
use super::helpers::stored_thread_from_rollout_item;
use crate::ReadThreadParams;
use crate::StoredThread;
@@ -38,6 +40,18 @@ pub(super) async fn read_thread(
.await)
{
let mut thread = stored_thread_from_sqlite_metadata(store, metadata).await;
if !params.include_history
&& let Some(rollout_path) = thread.rollout_path.clone()
&& let Ok(mut rollout_thread) = read_thread_from_rollout_path(store, rollout_path).await
&& rollout_thread.thread_id == thread_id
&& !rollout_thread.preview.is_empty()
{
if thread.name.is_some() {
rollout_thread.name = thread.name;
}
rollout_thread.git_info = thread.git_info;
thread = rollout_thread;
}
attach_history_if_requested(&mut thread, params.include_history).await?;
return Ok(thread);
}
@@ -222,7 +236,7 @@ async fn stored_thread_from_sqlite_metadata(
store: &LocalThreadStore,
metadata: ThreadMetadata,
) -> StoredThread {
let name = match distinct_title(&metadata) {
let name = match distinct_thread_metadata_title(&metadata) {
Some(title) => Some(title),
None => find_thread_name_by_id(store.config.codex_home.as_path(), &metadata.id)
.await
@@ -334,22 +348,6 @@ fn stored_thread_from_meta_line(
}
}
fn distinct_title(metadata: &ThreadMetadata) -> Option<String> {
let title = metadata.title.trim();
if title.is_empty() || metadata.first_user_message.as_deref().map(str::trim) == Some(title) {
None
} else {
Some(title.to_string())
}
}
fn set_thread_name_from_title(thread: &mut StoredThread, title: String) {
if title.trim().is_empty() || thread.preview.trim() == title.trim() {
return;
}
thread.name = Some(title);
}
fn parse_session_source(source: &str) -> SessionSource {
serde_json::from_str(source)
.or_else(|_| serde_json::from_value(serde_json::Value::String(source.to_string())))
@@ -374,6 +372,7 @@ fn parse_rfc3339_non_optional(value: &str) -> Option<DateTime<Utc>> {
#[cfg(test)]
mod tests {
use std::io::Write;
use std::path::PathBuf;
use chrono::Utc;
use codex_protocol::ThreadId;
@@ -636,6 +635,81 @@ mod tests {
assert_eq!(thread.name, Some("Saved title".to_string()));
}
#[tokio::test]
async fn read_thread_preserves_rollout_cwd_when_sqlite_metadata_exists() {
let home = TempDir::new().expect("temp dir");
let config = test_config(home.path());
let store = LocalThreadStore::new(config.clone());
let uuid = Uuid::from_u128(224);
let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id");
let day_dir = home.path().join("sessions/2025/01/03");
std::fs::create_dir_all(&day_dir).expect("sessions dir");
let rollout_path = day_dir.join(format!("rollout-2025-01-03T12-00-00-{uuid}.jsonl"));
let mut file = std::fs::File::create(&rollout_path).expect("session file");
let rollout_cwd = PathBuf::from("/");
let meta = serde_json::json!({
"timestamp": "2025-01-03T12:00:00Z",
"type": "session_meta",
"payload": {
"id": uuid,
"timestamp": "2025-01-03T12:00:00Z",
"cwd": rollout_cwd,
"originator": "test_originator",
"cli_version": "test_version",
"source": "cli",
"model_provider": "rollout-provider"
},
});
writeln!(file, "{meta}").expect("write session meta");
let user_event = serde_json::json!({
"timestamp": "2025-01-03T12:00:00Z",
"type": "event_msg",
"payload": {
"type": "user_message",
"message": "Hello from rollout",
"kind": "plain",
},
});
writeln!(file, "{user_event}").expect("write user event");
let runtime = codex_state::StateRuntime::init(
config.sqlite_home.clone(),
config.model_provider_id.clone(),
)
.await
.expect("state db should initialize");
let mut builder = ThreadMetadataBuilder::new(
thread_id,
rollout_path.clone(),
Utc::now(),
SessionSource::Cli,
);
builder.model_provider = Some(config.model_provider_id.clone());
builder.cwd = home.path().join("sqlite-workspace");
let mut metadata = builder.build(config.model_provider_id.as_str());
metadata.title = "Saved title".to_string();
metadata.first_user_message = Some("Hello from sqlite".to_string());
runtime
.upsert_thread(&metadata)
.await
.expect("state db upsert should succeed");
let thread = store
.read_thread(ReadThreadParams {
thread_id,
include_archived: false,
include_history: false,
})
.await
.expect("read thread");
assert_eq!(thread.thread_id, thread_id);
assert_eq!(thread.rollout_path, Some(rollout_path));
assert_eq!(thread.preview, "Hello from rollout");
assert_eq!(thread.name, Some("Saved title".to_string()));
assert_eq!(thread.cwd, rollout_cwd);
}
#[tokio::test]
async fn read_thread_uses_legacy_thread_name_when_sqlite_title_is_missing() {
let home = TempDir::new().expect("temp dir");