Files
codex/codex-rs/core/src/test_support.rs
T
Ahmed Ibrahim b02388672f Stabilize Windows cmd-based shell test harnesses (#14958)
## What is flaky
The Windows shell-driven integration tests in `codex-rs/core` were
intermittently unstable, especially:

- `apply_patch_cli_can_use_shell_command_output_as_patch_input`
- `websocket_test_codex_shell_chain`
- `websocket_v2_test_codex_shell_chain`

## Why it was flaky
These tests were exercising real shell-tool flows through whichever
shell Codex selected on Windows, and the `apply_patch` test also nested
a PowerShell read inside `cmd /c`.

There were multiple independent sources of nondeterminism in that setup:

- The test harness depended on the model-selected Windows shell instead
of pinning the shell it actually meant to exercise.
- `cmd.exe /c powershell.exe -Command "..."` is quoting-sensitive; on CI
that could leave the read command wrapped as a literal string instead of
executing it.
- Even after getting the quoting right, PowerShell could emit CLIXML
progress records like module-initialization output onto stdout.
- The `apply_patch` test was building a patch directly from shell
stdout, so any quoting artifact or progress noise corrupted the patch
input.

So the failures were driven by shell startup and output-shape variance,
not by the `apply_patch` or websocket logic themselves.

## How this PR fixes it
- Add a test-only `user_shell_override` path so Windows integration
tests can pin `cmd.exe` explicitly.
- Use that override in the websocket shell-chain tests and in the
`apply_patch` harness.
- Change the nested Windows file read in
`apply_patch_cli_can_use_shell_command_output_as_patch_input` to a UTF-8
PowerShell `-EncodedCommand` script.
- Run that nested PowerShell process with `-NonInteractive`, set
`$ProgressPreference = 'SilentlyContinue'`, and read the file with
`[System.IO.File]::ReadAllText(...)`.

## Why this fix fixes the flakiness
The outer harness now runs under a deterministic shell, and the inner
PowerShell read no longer depends on fragile `cmd` quoting or on
progress output staying quiet by accident. The shell tool returns only
the file contents, so patch construction and websocket assertions depend
on stable test inputs instead of on runner-specific shell behavior.

---------

Co-authored-by: Ahmed Ibrahim <219906144+aibrahim-oai@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
2026-03-17 20:21:46 +00:00

119 lines
3.8 KiB
Rust

//! Test-only helpers exposed for cross-crate integration tests.
//!
//! Production code should not depend on this module.
//! We prefer this to using a crate feature to avoid building multiple
//! permutations of the crate.
use std::path::PathBuf;
use std::sync::Arc;
use codex_protocol::config_types::CollaborationModeMask;
use codex_protocol::openai_models::ModelInfo;
use codex_protocol::openai_models::ModelPreset;
use codex_protocol::openai_models::ModelsResponse;
use once_cell::sync::Lazy;
use crate::AuthManager;
use crate::CodexAuth;
use crate::ModelProviderInfo;
use crate::ThreadManager;
use crate::config::Config;
use crate::models_manager::collaboration_mode_presets;
use crate::models_manager::manager::ModelsManager;
use crate::thread_manager;
use crate::unified_exec;
static TEST_MODEL_PRESETS: Lazy<Vec<ModelPreset>> = Lazy::new(|| {
let file_contents = include_str!("../models.json");
let mut response: ModelsResponse = serde_json::from_str(file_contents)
.unwrap_or_else(|err| panic!("bundled models.json should parse: {err}"));
response.models.sort_by(|a, b| a.priority.cmp(&b.priority));
let mut presets: Vec<ModelPreset> = response.models.into_iter().map(Into::into).collect();
ModelPreset::mark_default_by_picker_visibility(&mut presets);
presets
});
pub fn set_thread_manager_test_mode(enabled: bool) {
thread_manager::set_thread_manager_test_mode_for_tests(enabled);
}
pub fn set_deterministic_process_ids(enabled: bool) {
unified_exec::set_deterministic_process_ids_for_tests(enabled);
}
pub fn auth_manager_from_auth(auth: CodexAuth) -> Arc<AuthManager> {
AuthManager::from_auth_for_testing(auth)
}
pub fn auth_manager_from_auth_with_home(auth: CodexAuth, codex_home: PathBuf) -> Arc<AuthManager> {
AuthManager::from_auth_for_testing_with_home(auth, codex_home)
}
pub fn thread_manager_with_models_provider(
auth: CodexAuth,
provider: ModelProviderInfo,
) -> ThreadManager {
ThreadManager::with_models_provider_for_tests(auth, provider)
}
pub fn thread_manager_with_models_provider_and_home(
auth: CodexAuth,
provider: ModelProviderInfo,
codex_home: PathBuf,
) -> ThreadManager {
ThreadManager::with_models_provider_and_home_for_tests(auth, provider, codex_home)
}
pub async fn start_thread_with_user_shell_override(
thread_manager: &ThreadManager,
config: Config,
user_shell_override: crate::shell::Shell,
) -> crate::error::Result<crate::NewThread> {
thread_manager
.start_thread_with_user_shell_override_for_tests(config, user_shell_override)
.await
}
pub async fn resume_thread_from_rollout_with_user_shell_override(
thread_manager: &ThreadManager,
config: Config,
rollout_path: PathBuf,
auth_manager: Arc<AuthManager>,
user_shell_override: crate::shell::Shell,
) -> crate::error::Result<crate::NewThread> {
thread_manager
.resume_thread_from_rollout_with_user_shell_override_for_tests(
config,
rollout_path,
auth_manager,
user_shell_override,
)
.await
}
pub fn models_manager_with_provider(
codex_home: PathBuf,
auth_manager: Arc<AuthManager>,
provider: ModelProviderInfo,
) -> ModelsManager {
ModelsManager::with_provider_for_tests(codex_home, auth_manager, provider)
}
pub fn get_model_offline(model: Option<&str>) -> String {
ModelsManager::get_model_offline_for_tests(model)
}
pub fn construct_model_info_offline(model: &str, config: &Config) -> ModelInfo {
ModelsManager::construct_model_info_offline_for_tests(model, config)
}
pub fn all_model_presets() -> &'static Vec<ModelPreset> {
&TEST_MODEL_PRESETS
}
pub fn builtin_collaboration_mode_presets() -> Vec<CollaborationModeMask> {
collaboration_mode_presets::builtin_collaboration_mode_presets(
collaboration_mode_presets::CollaborationModesConfig::default(),
)
}