From 283bc4cf011047314b4804c0f1ccd06e4f6a95c5 Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Tue, 23 Jun 2026 18:06:29 -0700 Subject: [PATCH] test: add app-server auto environment helper (#29746) ## Why Start moving towards app-server tests defaulting to running against remote & foreign OS executors. To do so we need a point of indirection similar to core integration tests' `build_with_auto_env`, but with the flexibility of letting tests control environment registration if they need to. ## What This adds: - `TestAppServer::new_with_auto_env()` for constructing an app server with a default environment defined by the test runner (e.g. bazel) - `TestAppServer::auto_env_params()` for tests to easily acquire turn env params tailored to the automatic environment - `TestAppServer::send_thread_start_request_with_auto_env()` to make it easy for tests to start a thread using the automatic environment The above methods all fail if the test calling them has set up an environment where the automatic environment configuration conflicts with test-created state. ## Validation Adds a couple of basic smoke tests to the app-server test suite. Follow-ups will migrate more tests to use it. --- codex-rs/Cargo.lock | 1 + codex-rs/app-server/tests/common/Cargo.toml | 1 + .../tests/common/test_app_server.rs | 79 +++++++++ .../app-server/tests/suite/v2/auto_env.rs | 151 ++++++++++++++++++ codex-rs/app-server/tests/suite/v2/mod.rs | 1 + codex-rs/core/tests/common/test_codex.rs | 13 ++ codex-rs/core/tests/suite/remote_env.rs | 3 +- codex-rs/exec-server/src/lib.rs | 4 + 8 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 codex-rs/app-server/tests/suite/v2/auto_env.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 5a32a8e18..99e5d1525 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -402,6 +402,7 @@ dependencies = [ "codex-app-server-protocol", "codex-config", "codex-core", + "codex-exec-server", "codex-features", "codex-login", "codex-models-manager", diff --git a/codex-rs/app-server/tests/common/Cargo.toml b/codex-rs/app-server/tests/common/Cargo.toml index 5b245f40d..a1537208b 100644 --- a/codex-rs/app-server/tests/common/Cargo.toml +++ b/codex-rs/app-server/tests/common/Cargo.toml @@ -19,6 +19,7 @@ chrono = { workspace = true } codex-app-server-protocol = { workspace = true } codex-config = { workspace = true } codex-core = { workspace = true } +codex-exec-server = { workspace = true } codex-features = { workspace = true } codex-login = { workspace = true } codex-models-manager = { workspace = true } diff --git a/codex-rs/app-server/tests/common/test_app_server.rs b/codex-rs/app-server/tests/common/test_app_server.rs index 8c2efa35e..fbe43e768 100644 --- a/codex-rs/app-server/tests/common/test_app_server.rs +++ b/codex-rs/app-server/tests/common/test_app_server.rs @@ -12,6 +12,7 @@ use tokio::process::ChildStdin; use tokio::process::ChildStdout; use anyhow::Context; +use anyhow::ensure; use codex_app_server_protocol::AppsListParams; use codex_app_server_protocol::CancelLoginAccountParams; use codex_app_server_protocol::ClientInfo; @@ -107,11 +108,19 @@ use codex_app_server_protocol::ThreadTurnsListParams; use codex_app_server_protocol::ThreadUnarchiveParams; use codex_app_server_protocol::ThreadUnsubscribeParams; use codex_app_server_protocol::TurnCompletedNotification; +use codex_app_server_protocol::TurnEnvironmentParams; use codex_app_server_protocol::TurnInterruptParams; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnSteerParams; use codex_app_server_protocol::WindowsSandboxSetupStartParams; +use codex_exec_server::CODEX_EXEC_SERVER_NOISE_AUTH_TOKEN_ENV_VAR; +use codex_exec_server::CODEX_EXEC_SERVER_NOISE_CHATGPT_ACCOUNT_ID_ENV_VAR; +use codex_exec_server::CODEX_EXEC_SERVER_NOISE_ENVIRONMENT_ID_ENV_VAR; +use codex_exec_server::CODEX_EXEC_SERVER_NOISE_REGISTRY_URL_ENV_VAR; +use codex_exec_server::CODEX_EXEC_SERVER_URL_ENV_VAR; use codex_login::default_client::CODEX_INTERNAL_ORIGINATOR_OVERRIDE_ENV_VAR; +use core_test_support::test_codex::TestEnv; +use core_test_support::test_codex::test_env; use tokio::process::Command; pub struct TestAppServer { @@ -124,6 +133,7 @@ pub struct TestAppServer { stdin: Option, stdout: BufReader, pending_messages: VecDeque, + auto_env: Option, } pub const DEFAULT_CLIENT_NAME: &str = "codex-app-server-tests"; @@ -139,6 +149,59 @@ impl TestAppServer { Self::new_with_env_and_args(codex_home, &[], &[DISABLE_PLUGIN_STARTUP_TASKS_ARG]).await } + /// Starts an app server with the standard test environment and retains it + /// for the server's lifetime. + /// + /// Local test runs explicitly remove `CODEX_EXEC_SERVER_URL`; Docker- and + /// Wine-backed runs set it to the remote fixture URL. Use + /// [`Self::auto_env_params`] or + /// [`Self::send_thread_start_request_with_auto_env`] to select the matching + /// target-native cwd in a thread. Because `environments.toml` overrides the + /// URL-based configuration, this helper rejects a `codex_home` containing + /// that file. + pub async fn new_with_auto_env(codex_home: &Path) -> anyhow::Result { + let environments_toml = codex_home.join("environments.toml"); + ensure!( + !environments_toml + .try_exists() + .with_context(|| format!("check whether {} exists", environments_toml.display()))?, + "new_with_auto_env cannot be used when {} exists", + environments_toml.display() + ); + + let auto_env = test_env().await?; + // Noise registry configuration takes precedence over the URL-based + // provider, so clear inherited values to keep the selection hermetic. + let env_overrides = [ + ( + CODEX_EXEC_SERVER_URL_ENV_VAR, + auto_env.environment().exec_server_url(), + ), + (CODEX_EXEC_SERVER_NOISE_REGISTRY_URL_ENV_VAR, None), + (CODEX_EXEC_SERVER_NOISE_ENVIRONMENT_ID_ENV_VAR, None), + (CODEX_EXEC_SERVER_NOISE_AUTH_TOKEN_ENV_VAR, None), + (CODEX_EXEC_SERVER_NOISE_CHATGPT_ACCOUNT_ID_ENV_VAR, None), + ]; + let mut app_server = Self::new_with_env(codex_home, &env_overrides).await?; + app_server.auto_env = Some(auto_env); + Ok(app_server) + } + + /// Returns app-server protocol parameters for the automatically selected + /// test environment. Returns an error unless this server was created with + /// [`Self::new_with_auto_env`]. + pub fn auto_env_params(&self) -> anyhow::Result { + let selection = self + .auto_env + .as_ref() + .context("auto environment is unavailable; use TestAppServer::new_with_auto_env")? + .selection(); + Ok(TurnEnvironmentParams { + environment_id: selection.environment_id.clone(), + cwd: selection.cwd.clone().into(), + }) + } + pub async fn new_without_managed_config(codex_home: &Path) -> anyhow::Result { Self::new_with_env(codex_home, &[(DISABLE_MANAGED_CONFIG_ENV_VAR, Some("1"))]).await } @@ -273,6 +336,7 @@ impl TestAppServer { stdin: Some(stdin), stdout, pending_messages: VecDeque::new(), + auto_env: None, }) } @@ -449,6 +513,21 @@ impl TestAppServer { self.send_request("thread/start", params).await } + /// Sends a `thread/start` request selecting the environment provisioned by + /// [`Self::new_with_auto_env`]. Returns an error if `params` already select + /// environments so the caller cannot accidentally override the fixture. + pub async fn send_thread_start_request_with_auto_env( + &mut self, + mut params: ThreadStartParams, + ) -> anyhow::Result { + ensure!( + params.environments.is_none(), + "send_thread_start_request_with_auto_env requires params.environments to be omitted" + ); + params.environments = Some(vec![self.auto_env_params()?]); + self.send_thread_start_request(params).await + } + /// Send a `thread/resume` JSON-RPC request. pub async fn send_thread_resume_request( &mut self, diff --git a/codex-rs/app-server/tests/suite/v2/auto_env.rs b/codex-rs/app-server/tests/suite/v2/auto_env.rs new file mode 100644 index 000000000..70fa2c884 --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/auto_env.rs @@ -0,0 +1,151 @@ +use anyhow::Result; +use app_test_support::TestAppServer; +use app_test_support::create_final_assistant_message_sse_response; +use app_test_support::create_mock_responses_server_sequence; +use app_test_support::create_shell_command_sse_response; +use app_test_support::to_response; +use app_test_support::write_mock_responses_config_toml; +use codex_app_server_protocol::CommandExecutionStatus; +use codex_app_server_protocol::ItemCompletedNotification; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::UserInput as V2UserInput; +use pretty_assertions::assert_eq; +use std::collections::BTreeMap; +use std::time::Duration; +use tempfile::TempDir; +use tokio::time::timeout; + +const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); + +#[tokio::test] +async fn thread_start_with_auto_env_uses_fixture_cwd() -> Result<()> { + let responses = vec![ + create_shell_command_sse_response( + vec!["echo".to_string(), "auto-env-ok".to_string()], + /*workdir*/ None, + /*timeout_ms*/ None, + "cwd-call", + )?, + create_final_assistant_message_sse_response("done")?, + ]; + let server = create_mock_responses_server_sequence(responses).await; + let codex_home = TempDir::new()?; + write_mock_responses_config_toml( + codex_home.path(), + &server.uri(), + &BTreeMap::new(), + /*auto_compact_limit*/ 100_000, + /*requires_openai_auth*/ None, + "mock_provider", + "compact", + )?; + + let mut mcp = TestAppServer::new_with_auto_env(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + let expected_environment = mcp.auto_env_params()?; + + let err = mcp + .send_thread_start_request_with_auto_env(ThreadStartParams { + environments: Some(Vec::new()), + ..Default::default() + }) + .await + .expect_err("the auto-env helper should reject caller-supplied environments"); + assert_eq!( + err.to_string(), + "send_thread_start_request_with_auto_env requires params.environments to be omitted" + ); + + let request_id = mcp + .send_thread_start_request_with_auto_env(ThreadStartParams::default()) + .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(response)?; + + let request_id = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![V2UserInput::Text { + text: "report the current directory".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let _: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + + let command = timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let notification = mcp + .read_stream_until_notification_message("item/completed") + .await?; + let completed: ItemCompletedNotification = serde_json::from_value( + notification + .params + .expect("item/completed params must be present"), + )?; + if let ThreadItem::CommandExecution { .. } = completed.item { + return Ok::(completed.item); + } + } + }) + .await??; + let ThreadItem::CommandExecution { + cwd, + status, + exit_code, + .. + } = command + else { + unreachable!("loop returns only command execution items"); + }; + assert_eq!( + (cwd, status, exit_code), + ( + expected_environment.cwd, + CommandExecutionStatus::Completed, + Some(0) + ) + ); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + Ok(()) +} + +#[tokio::test] +async fn auto_env_rejects_explicit_environment_config() -> Result<()> { + let codex_home = TempDir::new()?; + std::fs::write(codex_home.path().join("environments.toml"), "")?; + + let result = TestAppServer::new_with_auto_env(codex_home.path()).await; + let Err(err) = result else { + anyhow::bail!("auto-env construction unexpectedly succeeded"); + }; + assert_eq!( + err.to_string(), + format!( + "new_with_auto_env cannot be used when {} exists", + codex_home.path().join("environments.toml").display() + ) + ); + + Ok(()) +} diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index c777142e8..bbd995275 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -2,6 +2,7 @@ mod account; mod analytics; mod app_list; mod attestation; +mod auto_env; mod client_metadata; mod collaboration_mode_list; #[cfg(unix)] diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index cb6c36311..e40f37e3c 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -121,6 +121,7 @@ pub struct TestEnv { environment: codex_exec_server::Environment, exec_server_url: Option, cwd: AbsolutePathBuf, + selection: TurnEnvironmentSelection, local_cwd_temp_dir: Option>, remote_container_name: Option, } @@ -131,10 +132,12 @@ impl TestEnv { let cwd = local_cwd_temp_dir.abs(); let environment = codex_exec_server::Environment::create_for_tests(/*exec_server_url*/ None)?; + let selection = local(cwd.clone()); Ok(Self { environment, exec_server_url: None, cwd, + selection, local_cwd_temp_dir: Some(local_cwd_temp_dir), remote_container_name: None, }) @@ -148,6 +151,11 @@ impl TestEnv { &self.environment } + /// Returns the environment and target-native cwd selected by the test harness. + pub fn selection(&self) -> &TurnEnvironmentSelection { + &self.selection + } + fn local_cwd_temp_dir(&self) -> Option> { self.local_cwd_temp_dir.clone() } @@ -180,6 +188,10 @@ pub async fn test_env() -> Result { /*sandbox*/ None, ) .await?; + let selection = TurnEnvironmentSelection { + environment_id: codex_exec_server::REMOTE_ENVIRONMENT_ID.to_string(), + cwd: cwd_uri.clone(), + }; let cwd = if remote_env == TestEnvironment::WineExec { // TODO(anp): Convert `Config::cwd` to `LegacyAppPathString` and remove this // compatibility projection. @@ -198,6 +210,7 @@ pub async fn test_env() -> Result { environment, exec_server_url: Some(websocket_url), cwd, + selection, local_cwd_temp_dir: None, remote_container_name: remote_env.docker_container_name().map(str::to_owned), }) diff --git a/codex-rs/core/tests/suite/remote_env.rs b/codex-rs/core/tests/suite/remote_env.rs index e665e86a0..af805ea1a 100644 --- a/codex-rs/core/tests/suite/remote_env.rs +++ b/codex-rs/core/tests/suite/remote_env.rs @@ -175,8 +175,7 @@ async fn remote_test_env_can_connect_and_use_filesystem() -> Result<()> { let test_env = test_env().await?; let file_system = test_env.environment().get_filesystem(); - let file_path_abs = test_env.cwd().join("remote-test-env-ok"); - let file_path_uri = PathUri::from_host_native_path(&file_path_abs)?; + let file_path_uri = test_env.selection().cwd.join("remote-test-env-ok")?; let payload = b"remote-test-env-ok".to_vec(); file_system diff --git a/codex-rs/exec-server/src/lib.rs b/codex-rs/exec-server/src/lib.rs index c861e47ba..9827e741d 100644 --- a/codex-rs/exec-server/src/lib.rs +++ b/codex-rs/exec-server/src/lib.rs @@ -51,6 +51,10 @@ pub use codex_file_system::FileSystemResult; pub use codex_file_system::FileSystemSandboxContext; pub use codex_file_system::ReadDirectoryEntry; pub use codex_file_system::RemoveOptions; +pub use environment::CODEX_EXEC_SERVER_NOISE_AUTH_TOKEN_ENV_VAR; +pub use environment::CODEX_EXEC_SERVER_NOISE_CHATGPT_ACCOUNT_ID_ENV_VAR; +pub use environment::CODEX_EXEC_SERVER_NOISE_ENVIRONMENT_ID_ENV_VAR; +pub use environment::CODEX_EXEC_SERVER_NOISE_REGISTRY_URL_ENV_VAR; pub use environment::CODEX_EXEC_SERVER_URL_ENV_VAR; pub use environment::Environment; pub use environment::EnvironmentManager;