mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
Adam Perry @ OpenAI
6e7ab52929
## Why `just test` currently uses the CI-oriented nextest profile, which serializes app-server integration tests even on developer machines that can run several safely. Bounded local parallelism substantially shortens this common iteration loop without changing CI behavior. Eight-worker experiments were faster, but keeping them reliable required relaxing several test deadlines. Four workers for integration tests is a solid tradeoff that speeds up local testing without needing to change test logic. ## What changed - Add a `local` nextest profile that inherits the existing defaults. - Allow up to four app-server integration tests to run concurrently under that profile. - Make `just test` select the local profile on Unix and Windows. - Keep the default CI profile serialized and leave all test deadlines unchanged. The tests use separate processes, randomized temporary `CODEX_HOME` directories, and ephemeral ports. The remaining shared constraints are system resources; each app-server also uses a multi-thread Tokio runtime, and fuzzy-search tests can create additional worker threads, so the local cap remains intentionally conservative. ## Performance and validation All measurements below are warm, execution-only app-server runs with nextest retries disabled. On the current rebased branch, an AMD EPYC 7763 machine with 16 logical CPUs and 62 GiB RAM completed three consecutive runs: | Run | Nextest time | Wall time | Result | | --- | ---: | ---: | --- | | 1 | 142.941s | 145.17s | 836/836 passed | | 2 | 143.402s | 145.59s | 836/836 passed | | 3 | 142.870s | 145.08s | 836/836 passed | The mean wall time was 145.28s. The slow-inventory, approval replay, and zsh-fork tests all passed with their original deadlines. Earlier measurements on the same Linux machine, before the suite grew, showed the scaling that motivated the change: | App-server concurrency | Nextest time | Result | | --- | ---: | --- | | 1 | 369.5s | 572/572 passed | | 2 | 194.5s | 572/572 passed | | 4 | 111.0s mean over 3 runs | 3/3 clean | Four workers reduced that execution time by about 70%, a roughly 3.3x speedup over serialization.
85 lines
3.5 KiB
TOML
85 lines
3.5 KiB
TOML
[profile.default]
|
|
# Retry once so one transient failure does not fail full-CI outright.
|
|
# Fanout keeps the full-CI shards moving without treating every >30s test as
|
|
# stuck. Keep this aligned with the broader timeout budget we give sharded CI.
|
|
slow-timeout = { period = "30s", terminate-after = 2 }
|
|
retries = 1
|
|
|
|
[profile.default.junit]
|
|
path = "junit.xml"
|
|
|
|
[profile.local]
|
|
inherits = "default"
|
|
|
|
[test-groups.app_server_protocol_codegen]
|
|
max-threads = 1
|
|
|
|
[test-groups.app_server_integration]
|
|
max-threads = 1
|
|
|
|
# Higher concurrency causes integration test timeouts under resource contention
|
|
# on common developer machines.
|
|
[test-groups.app_server_integration_local]
|
|
max-threads = 4
|
|
|
|
[test-groups.core_apply_patch_cli_integration]
|
|
max-threads = 1
|
|
|
|
[test-groups.windows_sandbox_legacy_sessions]
|
|
max-threads = 1
|
|
|
|
[test-groups.windows_process_heavy]
|
|
max-threads = 2
|
|
|
|
[[profile.default.overrides]]
|
|
# Do not add new tests here
|
|
filter = 'test(rmcp_client) | test(humanlike_typing_1000_chars_appears_live_no_placeholder)'
|
|
slow-timeout = { period = "1m", terminate-after = 4 }
|
|
|
|
[[profile.default.overrides]]
|
|
filter = 'test(approval_matrix_covers_all_modes)'
|
|
slow-timeout = { period = "30s", terminate-after = 2 }
|
|
|
|
[[profile.default.overrides]]
|
|
filter = 'package(codex-app-server-protocol) & (test(typescript_schema_fixtures_match_generated) | test(json_schema_fixtures_match_generated) | test(generate_ts_with_experimental_api_retains_experimental_entries) | test(generated_ts_optional_nullable_fields_only_in_params) | test(generate_json_filters_experimental_fields_and_methods))'
|
|
test-group = 'app_server_protocol_codegen'
|
|
|
|
[[profile.default.overrides]]
|
|
# These integration tests spawn a fresh app-server subprocess per case.
|
|
# Keep the library unit tests parallel.
|
|
filter = 'package(codex-app-server) & kind(test)'
|
|
test-group = 'app_server_integration'
|
|
|
|
[[profile.local.overrides]]
|
|
# Use up to four app-server subprocesses locally. The global nextest pool still
|
|
# limits this to the machine's logical CPU count.
|
|
filter = 'package(codex-app-server) & kind(test)'
|
|
test-group = 'app_server_integration_local'
|
|
|
|
[[profile.default.overrides]]
|
|
# These tests exercise full Codex turns and apply_patch execution, and they are
|
|
# sensitive to Windows runner process-startup stalls when many cases launch at once.
|
|
filter = 'package(codex-core) & kind(test) & test(apply_patch_cli)'
|
|
test-group = 'core_apply_patch_cli_integration'
|
|
|
|
[[profile.default.overrides]]
|
|
# These tests create restricted-token Windows child processes and private desktops.
|
|
# Serialize them to avoid exhausting Windows session/global desktop resources in CI.
|
|
filter = 'package(codex-windows-sandbox) & test(legacy_)'
|
|
test-group = 'windows_sandbox_legacy_sessions'
|
|
|
|
[[profile.default.overrides]]
|
|
# This Codex-home startup path still exceeded the broader Windows-heavy ceiling
|
|
# in both Windows full-CI lanes after contention was reduced.
|
|
platform = 'cfg(windows)'
|
|
filter = 'test(start_thread_uses_all_default_environments_from_codex_home)'
|
|
slow-timeout = { period = "1m", terminate-after = 2 }
|
|
|
|
[[profile.default.overrides]]
|
|
# These Windows-heavy tests spawn subprocesses, session files, or JSON-RPC
|
|
# clients and have been the dominant source of 30s full-CI timeouts.
|
|
platform = 'cfg(windows)'
|
|
filter = 'test(suite::resume::) | test(suite::cli_stream::) | test(suite::auth_env::) | test(start_thread_uses_all_default_environments_from_codex_home) | test(connect_stdio_command_initializes_json_rpc_client_on_windows)'
|
|
test-group = 'windows_process_heavy'
|
|
slow-timeout = { period = "45s", terminate-after = 2 }
|