npm: remove legacy package artifact synthesis (#23836)

## Why

`rust-release` now publishes `codex-package-<target>.tar.gz` as the
canonical native package payload. npm staging should consume those
archives directly instead of keeping legacy synthesis code that fetched
`rg`, copied standalone binaries, and rebuilt an approximate package
layout.

That also means the package builder should not know the internal shape
of `codex-package`. It should extract and copy the target payload
wholesale so future layout changes stay localized to the archive
producer.

The release job stages `codex`, `codex-responses-api-proxy`, and
`codex-sdk` together, so native artifact download should be filtered,
observable, and shared across component installs. Since that native
hydration is now only used by release staging, keeping a separate
`install_native_deps.py` CLI adds an extra wrapper without a real
caller.

## What Changed

- Removed legacy `codex-package` synthesis and related compatibility
flags from npm staging.
- Folded the remaining native artifact hydration code into
`scripts/stage_npm_packages.py` and deleted
`codex-cli/scripts/install_native_deps.py`.
- Made platform package staging copy the full extracted target directory
instead of enumerating package entries.
- Kept non-`codex-package` native components under their component
directory name instead of using a legacy destination map.
- Split native staging by component set while sharing one
workflow-artifact cache across the invocation.
- Changed workflow artifact download to select target artifacts by name,
print sizes/progress, and reuse cached artifacts.
- Removed the implicit `CI=true` default from `build_npm_package.py`;
local CI-shaped runs should set that environment explicitly.
- Kept `npm pack` cache/log output in its temporary directory so packing
does not write to the user npm cache.

## Verification

- `python3 -m py_compile scripts/stage_npm_packages.py
codex-cli/scripts/build_npm_package.py`
- `python3 -m unittest discover -s scripts/codex_package -p "test_*.py"`
- `scripts/stage_npm_packages.py --help`
- `codex-cli/scripts/build_npm_package.py --help`
- Ran the release-shaped staging command from `rust-release.yml` against
workflow run https://github.com/openai/codex/actions/runs/26240748758
with `CI=true` set locally to match GitHub Actions:

```sh
CI=true python3 ./scripts/stage_npm_packages.py \
  --release-version 0.133.0 \
  --workflow-url https://github.com/openai/codex/actions/runs/26240748758 \
  --package codex \
  --package codex-responses-api-proxy \
  --package codex-sdk
```

That completed successfully, downloaded only the six target artifacts
once, reused the cache for `codex-responses-api-proxy`, and produced all
nine npm tarballs. Generated tarballs and staging/artifact temp dirs
were cleaned afterward.
This commit is contained in:
Michael Bolin
2026-05-21 20:43:48 +00:00
committed by GitHub
parent 24faf49b2a
commit b20e969f23
5 changed files with 400 additions and 800 deletions
+378 -54
View File
@@ -2,20 +2,32 @@
"""Stage one or more Codex npm packages for release."""
import argparse
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import contextmanager
from dataclasses import dataclass
import importlib.util
import json
import os
import shutil
import subprocess
import tarfile
import tempfile
from pathlib import Path
from typing import Sequence
REPO_ROOT = Path(__file__).resolve().parent.parent
BUILD_SCRIPT = REPO_ROOT / "codex-cli" / "scripts" / "build_npm_package.py"
INSTALL_NATIVE_DEPS = REPO_ROOT / "codex-cli" / "scripts" / "install_native_deps.py"
WORKFLOW_NAME = ".github/workflows/rust-release.yml"
GITHUB_REPO = "openai/codex"
BINARY_TARGETS = (
"x86_64-unknown-linux-musl",
"aarch64-unknown-linux-musl",
"x86_64-apple-darwin",
"aarch64-apple-darwin",
"x86_64-pc-windows-msvc",
"aarch64-pc-windows-msvc",
)
_SPEC = importlib.util.spec_from_file_location("codex_build_npm_package", BUILD_SCRIPT)
if _SPEC is None or _SPEC.loader is None:
@@ -25,6 +37,48 @@ _SPEC.loader.exec_module(_BUILD_MODULE)
PACKAGE_NATIVE_COMPONENTS = getattr(_BUILD_MODULE, "PACKAGE_NATIVE_COMPONENTS", {})
PACKAGE_EXPANSIONS = getattr(_BUILD_MODULE, "PACKAGE_EXPANSIONS", {})
CODEX_PLATFORM_PACKAGES = getattr(_BUILD_MODULE, "CODEX_PLATFORM_PACKAGES", {})
CODEX_PACKAGE_COMPONENT = getattr(_BUILD_MODULE, "CODEX_PACKAGE_COMPONENT", "codex-package")
@dataclass(frozen=True)
class BinaryComponent:
artifact_prefix: str
dest_dir: str
binary_basename: str
@dataclass(frozen=True)
class WorkflowArtifact:
name: str
size_in_bytes: int
BINARY_COMPONENTS = {
"codex-responses-api-proxy": BinaryComponent(
artifact_prefix="codex-responses-api-proxy",
dest_dir="codex-responses-api-proxy",
binary_basename="codex-responses-api-proxy",
),
}
def _gha_enabled() -> bool:
return os.environ.get("GITHUB_ACTIONS") == "true"
def _gha_escape(value: str) -> str:
return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
@contextmanager
def _gha_group(title: str):
if _gha_enabled():
print(f"::group::{_gha_escape(title)}", flush=True)
try:
yield
finally:
if _gha_enabled():
print("::endgroup::", flush=True)
def parse_args() -> argparse.Namespace:
@@ -56,33 +110,23 @@ def parse_args() -> argparse.Namespace:
action="store_true",
help="Retain temporary staging directories instead of deleting them.",
)
parser.add_argument(
"--allow-missing-native-component",
dest="allow_missing_native_components",
action="append",
default=[],
help=(
"Native component that may be absent from reused workflow artifacts. "
"Intended for CI compatibility only; release staging should not use this."
),
)
parser.add_argument(
"--allow-legacy-codex-package",
action="store_true",
help=(
"Allow codex-package layouts to be synthesized from legacy per-binary "
"workflow artifacts. Intended for CI compatibility only; release staging "
"should not use this."
),
)
return parser.parse_args()
def collect_native_components(packages: list[str]) -> set[str]:
components: set[str] = set()
def native_components_for_package(package: str) -> tuple[str, ...]:
return tuple(sorted(PACKAGE_NATIVE_COMPONENTS.get(package, [])))
def collect_native_component_sets(packages: list[str]) -> list[tuple[str, ...]]:
component_sets: list[tuple[str, ...]] = []
seen: set[tuple[str, ...]] = set()
for package in packages:
components.update(PACKAGE_NATIVE_COMPONENTS.get(package, []))
return components
components = native_components_for_package(package)
if not components or components in seen:
continue
seen.add(components)
component_sets.append(components)
return component_sets
def expand_packages(packages: list[str]) -> list[str]:
@@ -131,23 +175,280 @@ def install_native_components(
workflow_url: str,
components: set[str],
vendor_root: Path,
*,
allow_legacy_codex_package: bool,
artifacts_dir: Path,
) -> None:
if not components:
return
cmd = [str(INSTALL_NATIVE_DEPS), "--workflow-url", workflow_url]
if allow_legacy_codex_package:
cmd.append("--allow-legacy-codex-package")
for component in sorted(components):
cmd.extend(["--component", component])
cmd.append(str(vendor_root))
run_command(cmd)
vendor_dir = vendor_root / "vendor"
vendor_dir.mkdir(parents=True, exist_ok=True)
workflow_id = workflow_url.rstrip("/").split("/")[-1]
print(f"Downloading native artifacts from workflow {workflow_id}...", flush=True)
with _gha_group(f"Download native artifacts from workflow {workflow_id}"):
artifacts_dir.mkdir(parents=True, exist_ok=True)
install_from_workflow_artifacts(
workflow_id,
artifacts_dir,
sorted(components),
vendor_dir,
)
print(f"Installed native dependencies into {vendor_dir}", flush=True)
def install_from_workflow_artifacts(
workflow_id: str,
artifacts_dir: Path,
components: Sequence[str],
vendor_dir: Path,
) -> None:
artifacts = select_target_artifacts(workflow_id, components)
download_artifacts(workflow_id, artifacts_dir, artifacts)
if CODEX_PACKAGE_COMPONENT in components:
install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS)
install_binary_components(
artifacts_dir,
vendor_dir,
[BINARY_COMPONENTS[name] for name in components if name in BINARY_COMPONENTS],
)
def select_target_artifacts(
workflow_id: str,
components: Sequence[str],
) -> list[WorkflowArtifact]:
needs_target_artifacts = CODEX_PACKAGE_COMPONENT in components or any(
component in BINARY_COMPONENTS for component in components
)
if not needs_target_artifacts:
return []
artifacts_by_name = {
artifact.name: artifact for artifact in list_workflow_artifacts(workflow_id)
}
selected_artifacts: list[WorkflowArtifact] = []
for target in BINARY_TARGETS:
for artifact_name in [target, f"{target}-unsigned"]:
artifact = artifacts_by_name.get(artifact_name)
if artifact is not None:
selected_artifacts.append(artifact)
break
else:
raise FileNotFoundError(
f"Expected workflow artifact not found for target {target}"
)
return selected_artifacts
def list_workflow_artifacts(workflow_id: str) -> list[WorkflowArtifact]:
stdout = subprocess.check_output(
[
"gh",
"api",
f"repos/{GITHUB_REPO}/actions/runs/{workflow_id}/artifacts",
"--paginate",
"--jq",
".artifacts[] | [.name, .size_in_bytes] | @tsv",
],
text=True,
)
artifacts: list[WorkflowArtifact] = []
for line in stdout.splitlines():
name, size_in_bytes = line.split("\t", 1)
artifacts.append(WorkflowArtifact(name=name, size_in_bytes=int(size_in_bytes)))
return artifacts
def download_artifacts(
workflow_id: str,
dest_dir: Path,
artifacts: Sequence[WorkflowArtifact],
) -> None:
total_bytes = sum(artifact.size_in_bytes for artifact in artifacts)
print(
f"Downloading {len(artifacts)} artifacts ({format_bytes(total_bytes)})",
flush=True,
)
for artifact in artifacts:
artifact_dir = dest_dir / artifact.name
if artifact_dir.is_dir() and any(artifact_dir.iterdir()):
print(
f" using cached {artifact.name} ({format_bytes(artifact.size_in_bytes)})",
flush=True,
)
continue
artifact_dir.mkdir(parents=True, exist_ok=True)
print(
f" downloading {artifact.name} ({format_bytes(artifact.size_in_bytes)})",
flush=True,
)
subprocess.check_call(
[
"gh",
"run",
"download",
"--name",
artifact.name,
"--dir",
str(artifact_dir),
"--repo",
GITHUB_REPO,
workflow_id,
]
)
def install_codex_package_archives(
artifacts_dir: Path,
vendor_dir: Path,
targets: Sequence[str],
) -> None:
if not targets:
return
print(
"Installing Codex package archives for targets: " + ", ".join(targets),
flush=True,
)
max_workers = min(len(targets), max(1, (os.cpu_count() or 1)))
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = {
executor.submit(
install_single_codex_package_archive,
artifacts_dir,
vendor_dir,
target,
): target
for target in targets
}
for future in as_completed(futures):
installed_path = future.result()
print(f" installed {installed_path}", flush=True)
def install_single_codex_package_archive(
artifacts_dir: Path,
vendor_dir: Path,
target: str,
) -> Path:
artifact_subdir = artifact_dir_for_target(artifacts_dir, target)
archive_path = artifact_subdir / f"codex-package-{target}.tar.gz"
if not archive_path.exists():
raise FileNotFoundError(f"Expected package archive not found: {archive_path}")
dest_dir = vendor_dir / target
if dest_dir.exists():
shutil.rmtree(dest_dir)
dest_dir.mkdir(parents=True, exist_ok=True)
with tarfile.open(archive_path, "r:gz") as archive:
archive.extractall(dest_dir, filter="data")
return dest_dir
def install_binary_components(
artifacts_dir: Path,
vendor_dir: Path,
selected_components: Sequence[BinaryComponent],
) -> None:
for component in selected_components:
component_targets = list(BINARY_TARGETS)
print(
f"Installing {component.binary_basename} binaries for targets: "
+ ", ".join(component_targets),
flush=True,
)
max_workers = min(len(component_targets), max(1, (os.cpu_count() or 1)))
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = {
executor.submit(
install_single_binary,
artifacts_dir,
vendor_dir,
target,
component,
): target
for target in component_targets
}
for future in as_completed(futures):
installed_path = future.result()
print(f" installed {installed_path}", flush=True)
def install_single_binary(
artifacts_dir: Path,
vendor_dir: Path,
target: str,
component: BinaryComponent,
) -> Path:
artifact_subdir = artifact_dir_for_target(artifacts_dir, target)
archive_path = binary_archive_path(artifact_subdir, component.artifact_prefix, target)
dest_dir = vendor_dir / target / component.dest_dir
dest_dir.mkdir(parents=True, exist_ok=True)
binary_name = (
f"{component.binary_basename}.exe" if "windows" in target else component.binary_basename
)
dest = dest_dir / binary_name
dest.unlink(missing_ok=True)
extract_zstd_archive(archive_path, dest)
if "windows" not in target:
dest.chmod(0o755)
return dest
def binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path:
archive_names = [archive_name_for_target(artifact_prefix, target)]
if artifact_dir.name == f"{target}-unsigned":
archive_names.append(archive_name_for_target(artifact_prefix, f"{target}-unsigned"))
for archive_name in archive_names:
archive_path = artifact_dir / archive_name
if archive_path.exists():
return archive_path
raise FileNotFoundError(f"Expected artifact not found: {artifact_dir / archive_names[0]}")
def archive_name_for_target(artifact_prefix: str, target: str) -> str:
if "windows" in target:
return f"{artifact_prefix}-{target}.exe.zst"
return f"{artifact_prefix}-{target}.zst"
def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path:
for artifact_name in [target, f"{target}-unsigned"]:
artifact_dir = artifacts_dir / artifact_name
if artifact_dir.is_dir():
return artifact_dir
return artifacts_dir / target
def extract_zstd_archive(archive_path: Path, dest: Path) -> None:
dest.parent.mkdir(parents=True, exist_ok=True)
output_path = archive_path.parent / dest.name
subprocess.check_call(["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)])
shutil.move(str(output_path), dest)
def format_bytes(size_in_bytes: int) -> str:
value = float(size_in_bytes)
for unit in ["B", "KiB", "MiB"]:
if value < 1024:
return f"{value:.1f} {unit}"
value /= 1024
return f"{value:.1f} GiB"
def run_command(cmd: list[str]) -> None:
print("+", " ".join(cmd))
print("+", " ".join(cmd), flush=True)
subprocess.run(cmd, cwd=REPO_ROOT, check=True)
@@ -167,36 +468,58 @@ def main() -> int:
runner_temp = Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir()))
packages = expand_packages(list(args.packages))
native_components = collect_native_components(packages)
allow_missing_native_components = set(args.allow_missing_native_components)
native_components_to_install = native_components - allow_missing_native_components
native_component_sets = collect_native_component_sets(packages)
print("Expanded packages: " + ", ".join(packages), flush=True)
if native_component_sets:
component_sets = [
"(" + ", ".join(components) + ")" for components in native_component_sets
]
print(
"Native component sets: " + ", ".join(component_sets),
flush=True,
)
vendor_temp_root: Path | None = None
vendor_src: Path | None = None
vendor_temp_roots: list[Path] = []
vendor_src_by_components: dict[tuple[str, ...], Path] = {}
artifacts_temp_root: Path | None = None
resolved_head_sha: str | None = None
final_messages = []
try:
if native_components_to_install:
if native_component_sets:
workflow_url, resolved_head_sha = resolve_workflow_url(
args.release_version, args.workflow_url
)
vendor_temp_root = Path(tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp))
install_native_components(
workflow_url,
native_components_to_install,
vendor_temp_root,
allow_legacy_codex_package=args.allow_legacy_codex_package,
print(f"Using native artifacts from {workflow_url}", flush=True)
artifacts_temp_root = Path(
tempfile.mkdtemp(prefix="npm-native-artifacts-", dir=runner_temp)
)
vendor_src = vendor_temp_root / "vendor"
print(f"Caching downloaded artifacts in {artifacts_temp_root}", flush=True)
for components in native_component_sets:
vendor_temp_root = Path(tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp))
vendor_temp_roots.append(vendor_temp_root)
print(
"Installing native components "
+ ", ".join(components)
+ f" into {vendor_temp_root}",
flush=True,
)
install_native_components(
workflow_url,
set(components),
vendor_temp_root,
artifacts_temp_root,
)
vendor_src_by_components[components] = vendor_temp_root / "vendor"
if resolved_head_sha:
print(f"should `git checkout {resolved_head_sha}`")
print(f"should `git checkout {resolved_head_sha}`", flush=True)
for package in packages:
staging_dir = Path(tempfile.mkdtemp(prefix=f"npm-stage-{package}-", dir=runner_temp))
pack_output = output_dir / tarball_name_for_package(package, args.release_version)
print(f"Staging {package} in {staging_dir}", flush=True)
cmd = [
str(BUILD_SCRIPT),
@@ -210,12 +533,10 @@ def main() -> int:
str(pack_output),
]
vendor_src = vendor_src_by_components.get(native_components_for_package(package))
if vendor_src is not None:
cmd.extend(["--vendor-src", str(vendor_src)])
for component in sorted(allow_missing_native_components):
cmd.extend(["--allow-missing-native-component", component])
try:
run_command(cmd)
finally:
@@ -224,11 +545,14 @@ def main() -> int:
final_messages.append(f"Staged {package} at {pack_output}")
finally:
if vendor_temp_root is not None and not args.keep_staging_dirs:
shutil.rmtree(vendor_temp_root, ignore_errors=True)
if not args.keep_staging_dirs:
for vendor_temp_root in vendor_temp_roots:
shutil.rmtree(vendor_temp_root, ignore_errors=True)
if artifacts_temp_root is not None:
shutil.rmtree(artifacts_temp_root, ignore_errors=True)
for msg in final_messages:
print(msg)
print(msg, flush=True)
return 0