mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
[codex] stage npm packages concurrently (#27853)
In the release job from https://github.com/openai/codex/actions/runs/27391514823 staging the nine npm release tarballs serially took 104 seconds. Each package build writes to a separate staging directory, output path, and npm cache. Run them through the script's existing thread pool, bounded by the available CPU count. Delete each staging tree as its build finishes so concurrency does not retain all copies until the end. On ubuntu-24.04 in https://github.com/openai/codex/actions/runs/27397232050 two serial trials took 103 and 101 seconds, while concurrent trials both took 41 seconds. Comparing every extracted file from the first serial and concurrent sets found no differences. This removes about one minute from every release.
This commit is contained in:
committed by
GitHub
Unverified
parent
51483bb583
commit
1460b509f4
@@ -503,8 +503,7 @@ def main() -> int:
|
||||
artifacts_temp_root: Path | None = None
|
||||
remove_artifacts_temp_root = False
|
||||
resolved_head_sha: str | None = None
|
||||
|
||||
final_messages = []
|
||||
staging_jobs: list[tuple[Path, list[str], str]] = []
|
||||
|
||||
try:
|
||||
if native_component_sets:
|
||||
@@ -570,22 +569,33 @@ def main() -> int:
|
||||
if vendor_src is not None:
|
||||
cmd.extend(["--vendor-src", str(vendor_src)])
|
||||
|
||||
try:
|
||||
run_command(cmd)
|
||||
finally:
|
||||
if not args.keep_staging_dirs:
|
||||
shutil.rmtree(staging_dir, ignore_errors=True)
|
||||
staging_jobs.append(
|
||||
(staging_dir, cmd, f"Staged {package} at {pack_output}")
|
||||
)
|
||||
|
||||
final_messages.append(f"Staged {package} at {pack_output}")
|
||||
max_workers = min(len(staging_jobs), os.cpu_count() or 1)
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {
|
||||
executor.submit(run_command, cmd): staging_dir
|
||||
for staging_dir, cmd, _message in staging_jobs
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
future.result()
|
||||
finally:
|
||||
if not args.keep_staging_dirs:
|
||||
shutil.rmtree(futures[future], ignore_errors=True)
|
||||
finally:
|
||||
if not args.keep_staging_dirs:
|
||||
for staging_dir, _cmd, _message in staging_jobs:
|
||||
shutil.rmtree(staging_dir, ignore_errors=True)
|
||||
for vendor_temp_root in vendor_temp_roots:
|
||||
shutil.rmtree(vendor_temp_root, ignore_errors=True)
|
||||
if remove_artifacts_temp_root and artifacts_temp_root is not None:
|
||||
shutil.rmtree(artifacts_temp_root, ignore_errors=True)
|
||||
|
||||
for msg in final_messages:
|
||||
print(msg, flush=True)
|
||||
for _staging_dir, _cmd, message in staging_jobs:
|
||||
print(message, flush=True)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user