mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
fix(app-server): avoid blocking connection cleanup (#26852)
## Why Remote-control app-server sessions can reconnect every 5-7 seconds when the shared transport-event queue fills. The queue's consumer handled `ConnectionClosed` by awaiting all in-flight RPCs for the disconnected connection. A stuck RPC therefore blocked processing of replacement connection and initialize events until remote-control forwarding hit its five-second timeout and reconnected again. Related issue: N/A (internal remote-control incident investigation). ## What Changed - Split fast RPC admission closure from draining: `ConnectionRpcGate::close()` rejects queued and future RPCs, while `shutdown()` continues waiting for RPCs that already started. - Close a disconnected connection's RPC gate before spawning the existing RPC drain and resource cleanup in a tracked background task, so the transport-event consumer remains available without waiting for active RPCs. - Reap completed cleanup tasks during normal operation, drain them during graceful shutdown, and abort them during forced shutdown. - Add regression coverage for closing with an active RPC, rejecting post-close requests without polling them, and preserving the existing shutdown wait behavior. ## Verification `just test -p codex-app-server --lib connection_rpc_gate` passes all 6 tests, including the new close-versus-drain regression coverage.
This commit is contained in:
@@ -20,6 +20,7 @@ use std::sync::atomic::AtomicBool;
|
||||
|
||||
use crate::analytics_utils::analytics_events_client_from_config;
|
||||
use crate::config_manager::ConfigManager;
|
||||
use crate::connection_cleanup::ConnectionCleanupTasks;
|
||||
use crate::message_processor::MessageProcessor;
|
||||
use crate::message_processor::MessageProcessorArgs;
|
||||
use crate::outgoing_message::ConnectionId;
|
||||
@@ -81,6 +82,7 @@ mod command_exec;
|
||||
mod config;
|
||||
mod config_manager;
|
||||
mod config_manager_service;
|
||||
mod connection_cleanup;
|
||||
mod connection_rpc_gate;
|
||||
mod dynamic_tools;
|
||||
mod error_code;
|
||||
@@ -819,6 +821,7 @@ pub async fn run_main_with_transport_options(
|
||||
let mut thread_created_rx = processor.thread_created_receiver();
|
||||
let mut running_turn_count_rx = processor.subscribe_running_assistant_turn_count();
|
||||
let mut connections = HashMap::<ConnectionId, ConnectionState>::new();
|
||||
let mut connection_cleanup_tasks = ConnectionCleanupTasks::new();
|
||||
let mut remote_control_status_rx = remote_control_handle.status_receiver();
|
||||
let mut remote_control_status = remote_control_status_rx.borrow().clone();
|
||||
let transport_shutdown_token = transport_shutdown_token.clone();
|
||||
@@ -906,14 +909,20 @@ pub async fn run_main_with_transport_options(
|
||||
let Some(connection_state) = connections.remove(&connection_id) else {
|
||||
continue;
|
||||
};
|
||||
if outbound_control_tx
|
||||
connection_state.session.rpc_gate.close().await;
|
||||
let outbound_closed = outbound_control_tx
|
||||
.send(OutboundControlEvent::Closed { connection_id })
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
.is_ok();
|
||||
let processor = Arc::clone(&processor);
|
||||
connection_cleanup_tasks.spawn(async move {
|
||||
processor
|
||||
.connection_closed(connection_id, &connection_state.session)
|
||||
.await;
|
||||
});
|
||||
if !outbound_closed {
|
||||
break;
|
||||
}
|
||||
processor.connection_closed(connection_id, &connection_state.session).await;
|
||||
if shutdown_when_no_connections && connections.is_empty() {
|
||||
break;
|
||||
}
|
||||
@@ -1010,6 +1019,7 @@ pub async fn run_main_with_transport_options(
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = connection_cleanup_tasks.reap_next() => {}
|
||||
changed = remote_control_status_rx.changed() => {
|
||||
if changed.is_err() {
|
||||
continue;
|
||||
@@ -1062,8 +1072,11 @@ pub async fn run_main_with_transport_options(
|
||||
.map(|connection_state| connection_state.session.rpc_gate.shutdown()),
|
||||
)
|
||||
.await;
|
||||
connection_cleanup_tasks.drain().await;
|
||||
processor.drain_background_tasks().await;
|
||||
processor.shutdown_threads().await;
|
||||
} else {
|
||||
connection_cleanup_tasks.abort();
|
||||
}
|
||||
info!("processor task exited (channel closed)");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user