[2/4] Implement executor HTTP request runner (#18582)

### Why
Remote streamable HTTP MCP needs the executor to perform ordinary HTTP
requests on the executor side. This keeps network placement aligned with
`experimental_environment = "remote"` without adding MCP-specific
executor APIs.

### What
- Add an executor-side `http/request` runner backed by `reqwest`.
- Validate request method and URL scheme, preserving the transport
boundary at plain HTTP.
- Return buffered responses for ordinary calls and emit ordered
`http/request/bodyDelta` notifications for streaming responses.
- Register the request handler in the exec-server router.
- Document the runner entrypoint, conversion helpers, body-stream
bridge, notification sender, timeout behavior, and new integration-test
helpers.
- Add exec-server integration tests with the existing websocket harness
and a local TCP HTTP peer for buffered and streamed responses, with
comments spelling out what each test proves and its
setup/exercise/assert phases.

### Stack
1. #18581 protocol
2. #18582 runner
3. #18583 RMCP client
4. #18584 manager wiring and local/remote coverage

### Verification
- `just fmt`
- `cargo check -p codex-exec-server -p codex-rmcp-client --tests`
- `cargo check -p codex-core --test all` compile-only
- `git diff --check`
- Online full CI is running from the `full-ci` branch, including the
remote Rust test job.

Co-authored-by: Codex <noreply@openai.com>

---------

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Ahmed Ibrahim
2026-04-22 20:36:34 +00:00
committed by GitHub
co-authored by Codex
parent 18a26d7bbc
commit 9360f267f3
10 changed files with 1169 additions and 105 deletions
+271 -71
View File
@@ -1,7 +1,15 @@
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Duration;
use codex_app_server_protocol::JSONRPCErrorError;
use futures::StreamExt;
use reqwest::Method;
use reqwest::Url;
use reqwest::header::HeaderMap;
use reqwest::header::HeaderName;
use reqwest::header::HeaderValue;
use serde_json::Value;
use serde_json::from_value;
use tokio::runtime::Handle;
@@ -12,14 +20,28 @@ use tracing::debug;
use super::ExecServerClient;
use super::ExecServerError;
use super::Inner;
use crate::protocol::HTTP_REQUEST_BODY_DELTA_METHOD;
use crate::protocol::HTTP_REQUEST_METHOD;
use crate::protocol::HttpHeader;
use crate::protocol::HttpRequestBodyDeltaNotification;
use crate::protocol::HttpRequestParams;
use crate::protocol::HttpRequestResponse;
use crate::rpc::RpcNotificationSender;
use crate::rpc::internal_error;
use crate::rpc::invalid_params;
/// Maximum queued body frames per streamed executor HTTP response.
const HTTP_BODY_DELTA_CHANNEL_CAPACITY: usize = 256;
pub(crate) struct ExecutorPendingHttpBodyStream {
pub(crate) request_id: String,
response: reqwest::Response,
}
pub(crate) struct ExecutorHttpRequestRunner {
client: reqwest::Client,
}
/// Request-scoped stream of body chunks for an executor HTTP response.
///
/// The initial `http/request` call returns status and headers. This stream then
@@ -35,6 +57,60 @@ pub struct HttpResponseBodyStream {
closed: bool,
}
impl ExecServerClient {
/// Performs an executor-side HTTP request and buffers the response body.
pub async fn http_request(
&self,
mut params: HttpRequestParams,
) -> Result<HttpRequestResponse, ExecServerError> {
params.stream_response = false;
self.call(HTTP_REQUEST_METHOD, &params).await
}
/// Performs an executor-side HTTP request and returns a body stream.
///
/// The method sets `stream_response` and replaces any caller-supplied
/// `request_id` with a connection-local id, so late deltas from abandoned
/// streams cannot be confused with later requests.
pub async fn http_request_stream(
&self,
mut params: HttpRequestParams,
) -> Result<(HttpRequestResponse, HttpResponseBodyStream), ExecServerError> {
params.stream_response = true;
let request_id = self.inner.next_http_body_stream_request_id();
params.request_id = request_id.clone();
let (tx, rx) = mpsc::channel(HTTP_BODY_DELTA_CHANNEL_CAPACITY);
self.inner
.insert_http_body_stream(request_id.clone(), tx)
.await?;
let mut registration = HttpBodyStreamRegistration {
inner: Arc::clone(&self.inner),
request_id: request_id.clone(),
active: true,
};
let response = match self.call(HTTP_REQUEST_METHOD, &params).await {
Ok(response) => response,
Err(error) => {
self.inner.remove_http_body_stream(&request_id).await;
registration.active = false;
return Err(error);
}
};
registration.active = false;
Ok((
response,
HttpResponseBodyStream {
inner: Arc::clone(&self.inner),
request_id,
next_seq: 1,
rx,
pending_eof: false,
closed: false,
},
))
}
}
impl HttpResponseBodyStream {
/// Receives the next response-body chunk.
///
@@ -109,75 +185,165 @@ impl Drop for HttpResponseBodyStream {
}
}
/// Active route registration owned while `http_request_stream` awaits headers.
struct HttpBodyStreamRegistration {
inner: Arc<Inner>,
request_id: String,
active: bool,
}
impl Drop for HttpBodyStreamRegistration {
/// Removes the route if the stream request future is cancelled before headers return.
fn drop(&mut self) {
if self.active {
spawn_remove_http_body_stream(Arc::clone(&self.inner), self.request_id.clone());
}
}
}
impl ExecServerClient {
/// Performs an executor-side HTTP request and buffers the response body.
pub async fn http_request(
&self,
mut params: HttpRequestParams,
) -> Result<HttpRequestResponse, ExecServerError> {
params.stream_response = false;
params.request_id = None;
self.call(HTTP_REQUEST_METHOD, &params).await
}
/// Performs an executor-side HTTP request and returns a body stream.
///
/// The method sets `stream_response` and replaces any caller-supplied
/// `request_id` with a connection-local id, so late deltas from abandoned
/// streams cannot be confused with later requests.
pub async fn http_request_stream(
&self,
mut params: HttpRequestParams,
) -> Result<(HttpRequestResponse, HttpResponseBodyStream), ExecServerError> {
params.stream_response = true;
let request_id = self.inner.next_http_body_stream_request_id();
params.request_id = Some(request_id.clone());
let (tx, rx) = mpsc::channel(HTTP_BODY_DELTA_CHANNEL_CAPACITY);
self.inner
.insert_http_body_stream(request_id.clone(), tx)
.await?;
let mut registration = HttpBodyStreamRegistration {
inner: Arc::clone(&self.inner),
request_id: request_id.clone(),
active: true,
};
let response = match self.call(HTTP_REQUEST_METHOD, &params).await {
Ok(response) => response,
Err(error) => {
self.inner.remove_http_body_stream(&request_id).await;
registration.active = false;
return Err(error);
impl ExecutorHttpRequestRunner {
pub(crate) fn new(timeout_ms: Option<u64>) -> Result<Self, JSONRPCErrorError> {
let client = match timeout_ms {
None => reqwest::Client::builder(),
Some(timeout_ms) => {
reqwest::Client::builder().timeout(Duration::from_millis(timeout_ms))
}
};
registration.active = false;
}
.build()
.map_err(|err| internal_error(format!("failed to build http/request client: {err}")))?;
Ok(Self { client })
}
pub(crate) async fn run(
&self,
params: HttpRequestParams,
) -> Result<(HttpRequestResponse, Option<ExecutorPendingHttpBodyStream>), JSONRPCErrorError>
{
let method = Method::from_bytes(params.method.as_bytes())
.map_err(|err| invalid_params(format!("http/request method is invalid: {err}")))?;
let url = Url::parse(&params.url)
.map_err(|err| invalid_params(format!("http/request url is invalid: {err}")))?;
match url.scheme() {
"http" | "https" => {}
scheme => {
return Err(invalid_params(format!(
"http/request only supports http and https URLs, got {scheme}"
)));
}
}
let headers = Self::build_headers(params.headers)?;
let mut request = self.client.request(method, url).headers(headers);
if let Some(body) = params.body {
request = request.body(body.into_inner());
}
let response = request
.send()
.await
.map_err(|err| internal_error(format!("http/request failed: {err}")))?;
let status = response.status().as_u16();
let headers = Self::response_headers(response.headers());
if params.stream_response {
return Ok((
HttpRequestResponse {
status,
headers,
body: Vec::new().into(),
},
Some(ExecutorPendingHttpBodyStream {
request_id: params.request_id,
response,
}),
));
}
let body = response.bytes().await.map_err(|err| {
internal_error(format!("failed to read http/request response body: {err}"))
})?;
Ok((
response,
HttpResponseBodyStream {
inner: Arc::clone(&self.inner),
request_id,
next_seq: 1,
rx,
pending_eof: false,
closed: false,
HttpRequestResponse {
status,
headers,
body: body.to_vec().into(),
},
None,
))
}
fn build_headers(headers: Vec<HttpHeader>) -> Result<HeaderMap, JSONRPCErrorError> {
let mut header_map = HeaderMap::new();
for header in headers {
let name = HeaderName::from_bytes(header.name.as_bytes()).map_err(|err| {
invalid_params(format!("http/request header name is invalid: {err}"))
})?;
let value = HeaderValue::from_str(&header.value).map_err(|err| {
invalid_params(format!(
"http/request header value is invalid for {}: {err}",
header.name
))
})?;
header_map.append(name, value);
}
Ok(header_map)
}
fn response_headers(headers: &HeaderMap) -> Vec<HttpHeader> {
headers
.iter()
.filter_map(|(name, value)| {
Some(HttpHeader {
name: name.as_str().to_string(),
value: value.to_str().ok()?.to_string(),
})
})
.collect()
}
pub(crate) async fn stream_body(
pending_stream: ExecutorPendingHttpBodyStream,
notifications: RpcNotificationSender,
) {
let ExecutorPendingHttpBodyStream {
request_id,
response,
} = pending_stream;
let mut seq = 1;
let mut body = response.bytes_stream();
while let Some(chunk) = body.next().await {
match chunk {
Ok(bytes) => {
if !send_executor_body_delta(
&notifications,
HttpRequestBodyDeltaNotification {
request_id: request_id.clone(),
seq,
delta: bytes.to_vec().into(),
done: false,
error: None,
},
)
.await
{
return;
}
seq += 1;
}
Err(err) => {
let _ = send_executor_body_delta(
&notifications,
HttpRequestBodyDeltaNotification {
request_id,
seq,
delta: Vec::new().into(),
done: true,
error: Some(err.to_string()),
},
)
.await;
return;
}
}
}
let _ = send_executor_body_delta(
&notifications,
HttpRequestBodyDeltaNotification {
request_id,
seq,
delta: Vec::new().into(),
done: true,
error: None,
},
)
.await;
}
}
impl Inner {
@@ -231,13 +397,21 @@ impl Inner {
let streams = streams.as_ref().clone();
self.http_body_streams.store(Arc::new(HashMap::new()));
for (request_id, tx) in streams {
let _ = tx.try_send(HttpRequestBodyDeltaNotification {
request_id,
seq: 1,
delta: Vec::new().into(),
done: true,
error: Some(message.clone()),
});
if tx
.try_send(HttpRequestBodyDeltaNotification {
request_id: request_id.clone(),
seq: 1,
delta: Vec::new().into(),
done: true,
error: Some(message.clone()),
})
.is_err()
{
let mut next_failures = self.http_body_stream_failures.load().as_ref().clone();
next_failures.insert(request_id, message.clone());
self.http_body_stream_failures
.store(Arc::new(next_failures));
}
}
}
@@ -312,6 +486,22 @@ impl Inner {
}
}
/// Active route registration owned while `http_request_stream` awaits headers.
struct HttpBodyStreamRegistration {
inner: Arc<Inner>,
request_id: String,
active: bool,
}
impl Drop for HttpBodyStreamRegistration {
/// Removes the route if the stream request future is cancelled before headers return.
fn drop(&mut self) {
if self.active {
spawn_remove_http_body_stream(Arc::clone(&self.inner), self.request_id.clone());
}
}
}
/// Schedules HTTP body route removal from synchronous drop paths.
fn spawn_remove_http_body_stream(inner: Arc<Inner>, request_id: String) {
if let Ok(handle) = Handle::try_current() {
@@ -320,3 +510,13 @@ fn spawn_remove_http_body_stream(inner: Arc<Inner>, request_id: String) {
});
}
}
async fn send_executor_body_delta(
notifications: &RpcNotificationSender,
delta: HttpRequestBodyDeltaNotification,
) -> bool {
notifications
.notify(HTTP_REQUEST_BODY_DELTA_METHOD, &delta)
.await
.is_ok()
}
+55 -5
View File
@@ -286,15 +286,19 @@ pub struct HttpRequestParams {
/// Optional request body bytes.
#[serde(default, rename = "bodyBase64")]
pub body: Option<ByteChunk>,
/// Optional request timeout in milliseconds.
#[serde(default)]
/// Request timeout in milliseconds.
///
/// Omitted or `null` disables the timeout. A number applies that exact
/// millisecond deadline.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub timeout_ms: Option<u64>,
/// Caller-chosen stream id for `http/request/bodyDelta` notifications.
///
/// The id must remain unique on a connection until the terminal body delta
/// arrives, even if the caller stops reading the stream earlier.
#[serde(default)]
pub request_id: Option<String>,
/// arrives, even if the caller stops reading the stream earlier. Buffered
/// requests still send an id so callers can keep one consistent request
/// envelope shape.
pub request_id: String,
/// Return after response headers and stream the response body as deltas.
#[serde(default)]
pub stream_response: bool,
@@ -391,3 +395,49 @@ mod base64_bytes {
.map_err(serde::de::Error::custom)
}
}
#[cfg(test)]
mod tests {
use super::HttpRequestParams;
use pretty_assertions::assert_eq;
#[test]
fn http_request_timeout_treats_omitted_and_null_as_no_timeout() {
let omitted: HttpRequestParams = serde_json::from_value(serde_json::json!({
"method": "GET",
"url": "https://example.test",
"requestId": "req-omitted-timeout",
}))
.expect("omitted timeout should deserialize");
let null_timeout: HttpRequestParams = serde_json::from_value(serde_json::json!({
"method": "GET",
"url": "https://example.test",
"requestId": "req-null-timeout",
"timeoutMs": null,
}))
.expect("null timeout should deserialize");
let explicit_timeout: HttpRequestParams = serde_json::from_value(serde_json::json!({
"method": "GET",
"url": "https://example.test",
"requestId": "req-explicit-timeout",
"timeoutMs": 1234,
}))
.expect("numeric timeout should deserialize");
assert_eq!(
(omitted.request_id.as_str(), omitted.timeout_ms),
("req-omitted-timeout", None)
);
assert_eq!(
(null_timeout.request_id.as_str(), null_timeout.timeout_ms),
("req-null-timeout", None)
);
assert_eq!(
(
explicit_timeout.request_id.as_str(),
explicit_timeout.timeout_ms
),
("req-explicit-timeout", Some(1234))
);
}
}
+44 -4
View File
@@ -36,8 +36,9 @@ pub(crate) enum RpcCallError {
type PendingRequest = oneshot::Sender<Result<Value, RpcCallError>>;
type BoxFuture<T> = Pin<Box<dyn Future<Output = T> + Send + 'static>>;
type RequestRoute<S> =
Box<dyn Fn(Arc<S>, JSONRPCRequest) -> BoxFuture<RpcServerOutboundMessage> + Send + Sync>;
type RequestRoute<S> = Box<
dyn Fn(Arc<S>, JSONRPCRequest) -> BoxFuture<Option<RpcServerOutboundMessage>> + Send + Sync,
>;
type NotificationRoute<S> =
Box<dyn Fn(Arc<S>, JSONRPCNotification) -> BoxFuture<Result<(), String>> + Send + Sync>;
@@ -72,6 +73,17 @@ impl RpcNotificationSender {
Self { outgoing_tx }
}
pub(crate) async fn response(
&self,
request_id: RequestId,
result: Value,
) -> Result<(), JSONRPCErrorError> {
self.outgoing_tx
.send(RpcServerOutboundMessage::Response { request_id, result })
.await
.map_err(|_| internal_error("RPC connection closed while sending response".into()))
}
#[allow(dead_code)]
pub(crate) async fn notify<P: Serialize>(
&self,
@@ -131,10 +143,10 @@ where
let response = match response {
Ok(response) => response.await,
Err(error) => {
return RpcServerOutboundMessage::Error { request_id, error };
return Some(RpcServerOutboundMessage::Error { request_id, error });
}
};
match response {
Some(match response {
Ok(result) => match serde_json::to_value(result) {
Ok(result) => RpcServerOutboundMessage::Response { request_id, result },
Err(err) => RpcServerOutboundMessage::Error {
@@ -143,6 +155,34 @@ where
},
},
Err(error) => RpcServerOutboundMessage::Error { request_id, error },
})
})
}),
);
}
pub(crate) fn request_with_id<P, F, Fut>(&mut self, method: &'static str, handler: F)
where
P: DeserializeOwned + Send + 'static,
F: Fn(Arc<S>, RequestId, P) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<(), JSONRPCErrorError>> + Send + 'static,
{
self.request_routes.insert(
method,
Box::new(move |state, request| {
let request_id = request.id;
let params = decode_request_params::<P>(request.params)
.map(|params| handler(state, request_id.clone(), params));
Box::pin(async move {
let response = match params {
Ok(response) => response.await,
Err(error) => {
return Some(RpcServerOutboundMessage::Error { request_id, error });
}
};
match response {
Ok(()) => None,
Err(error) => Some(RpcServerOutboundMessage::Error { request_id, error }),
}
})
}),
@@ -4,8 +4,16 @@ use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use codex_app_server_protocol::JSONRPCErrorError;
use codex_app_server_protocol::RequestId;
use serde_json::to_value;
use std::collections::HashSet;
use tokio::sync::Mutex;
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
use crate::ExecServerRuntimePaths;
use crate::client::http_client::ExecutorHttpRequestRunner;
use crate::client::http_client::ExecutorPendingHttpBodyStream;
use crate::protocol::ExecParams;
use crate::protocol::ExecResponse;
use crate::protocol::FsCopyParams;
@@ -22,6 +30,7 @@ use crate::protocol::FsRemoveParams;
use crate::protocol::FsRemoveResponse;
use crate::protocol::FsWriteFileParams;
use crate::protocol::FsWriteFileResponse;
use crate::protocol::HttpRequestParams;
use crate::protocol::InitializeParams;
use crate::protocol::InitializeResponse;
use crate::protocol::ReadParams;
@@ -31,6 +40,8 @@ use crate::protocol::TerminateResponse;
use crate::protocol::WriteParams;
use crate::protocol::WriteResponse;
use crate::rpc::RpcNotificationSender;
use crate::rpc::internal_error;
use crate::rpc::invalid_params;
use crate::rpc::invalid_request;
use crate::server::file_system_handler::FileSystemHandler;
use crate::server::session_registry::SessionHandle;
@@ -40,6 +51,9 @@ pub(crate) struct ExecServerHandler {
session_registry: Arc<SessionRegistry>,
notifications: RpcNotificationSender,
session: StdMutex<Option<SessionHandle>>,
active_body_stream_ids: Mutex<HashSet<String>>,
background_task_shutdown: CancellationToken,
background_tasks: TaskTracker,
file_system: FileSystemHandler,
initialize_requested: AtomicBool,
initialized: AtomicBool,
@@ -55,6 +69,9 @@ impl ExecServerHandler {
session_registry,
notifications,
session: StdMutex::new(None),
active_body_stream_ids: Mutex::new(HashSet::new()),
background_task_shutdown: CancellationToken::new(),
background_tasks: TaskTracker::new(),
file_system: FileSystemHandler::new(runtime_paths),
initialize_requested: AtomicBool::new(false),
initialized: AtomicBool::new(false),
@@ -62,6 +79,9 @@ impl ExecServerHandler {
}
pub(crate) async fn shutdown(&self) {
self.background_task_shutdown.cancel();
self.background_tasks.close();
self.background_tasks.wait().await;
if let Some(session) = self.session() {
session.detach().await;
}
@@ -147,6 +167,47 @@ impl ExecServerHandler {
session.process().terminate(params).await
}
pub(crate) async fn http_request(
self: &Arc<Self>,
request_id: RequestId,
params: HttpRequestParams,
) -> Result<(), JSONRPCErrorError> {
self.require_initialized_for("http")?;
let stream_response = params.stream_response;
let http_request_id = params.request_id.clone();
if stream_response {
self.reserve_http_body_stream(&http_request_id).await?;
}
let response = ExecutorHttpRequestRunner::new(params.timeout_ms)?
.run(params)
.await;
if response.is_err() && stream_response {
self.release_http_body_stream(&http_request_id).await;
}
let (response, mut pending_stream) = response?;
let result = match to_value(response) {
Ok(result) => result,
Err(err) => {
if let Some(pending_stream) = pending_stream.take() {
self.release_http_body_stream(&pending_stream.request_id)
.await;
}
return Err(internal_error(err.to_string()));
}
};
if let Err(error) = self.notifications.response(request_id, result).await {
if let Some(pending_stream) = pending_stream.take() {
self.release_http_body_stream(&pending_stream.request_id)
.await;
}
return Err(error);
}
if let Some(pending_stream) = pending_stream {
self.start_http_body_stream(pending_stream).await;
}
Ok(())
}
pub(crate) async fn fs_read_file(
&self,
params: FsReadFileParams,
@@ -242,6 +303,44 @@ impl ExecServerHandler {
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
async fn start_http_body_stream(
self: &Arc<Self>,
pending_stream: ExecutorPendingHttpBodyStream,
) {
let request_id = pending_stream.request_id.clone();
if self.background_task_shutdown.is_cancelled() {
self.release_http_body_stream(&request_id).await;
return;
}
let finished_request_id = request_id.clone();
let handler = Arc::clone(self);
let notifications = self.notifications.clone();
let shutdown = self.background_task_shutdown.clone();
self.background_tasks.spawn(async move {
tokio::select! {
_ = shutdown.cancelled() => {}
_ = ExecutorHttpRequestRunner::stream_body(pending_stream, notifications) => {}
}
handler.release_http_body_stream(&finished_request_id).await;
});
}
async fn release_http_body_stream(&self, request_id: &str) {
let mut active_body_stream_ids = self.active_body_stream_ids.lock().await;
active_body_stream_ids.remove(request_id);
}
async fn reserve_http_body_stream(&self, request_id: &str) -> Result<(), JSONRPCErrorError> {
let mut active_body_stream_ids = self.active_body_stream_ids.lock().await;
if active_body_stream_ids.contains(request_id) {
return Err(invalid_params(format!(
"http/request streamResponse requestId `{request_id}` is already active"
)));
}
active_body_stream_ids.insert(request_id.to_string());
Ok(())
}
}
#[cfg(test)]
+3 -1
View File
@@ -103,7 +103,9 @@ async fn run_connection(
break;
}
};
if outgoing_tx.send(message).await.is_err() {
if let Some(message) = message
&& outgoing_tx.send(message).await.is_err()
{
break;
}
} else if outgoing_tx
@@ -19,6 +19,8 @@ use crate::protocol::FsReadDirectoryParams;
use crate::protocol::FsReadFileParams;
use crate::protocol::FsRemoveParams;
use crate::protocol::FsWriteFileParams;
use crate::protocol::HTTP_REQUEST_METHOD;
use crate::protocol::HttpRequestParams;
use crate::protocol::INITIALIZE_METHOD;
use crate::protocol::INITIALIZED_METHOD;
use crate::protocol::InitializeParams;
@@ -42,6 +44,12 @@ pub(crate) fn build_router() -> RpcRouter<ExecServerHandler> {
handler.initialize(params).await
},
);
router.request_with_id(
HTTP_REQUEST_METHOD,
|handler: Arc<ExecServerHandler>, request_id, params: HttpRequestParams| async move {
handler.http_request(request_id, params).await
},
);
router.request(
EXEC_METHOD,
|handler: Arc<ExecServerHandler>, params: ExecParams| async move { handler.exec(params).await },