From 6d0e313e237b6c1dd055fc9b1d7469961e8c02f8 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Mon, 8 Jun 2026 10:16:36 -0700 Subject: [PATCH] [codex] Restore release symbol artifacts with line tables (#26202) ## Summary - Restore separate release symbol archives for macOS, Linux, and Windows binaries. - Build release binaries with `line-tables-only` debuginfo instead of full debuginfo. - Strip Unix distribution binaries after extracting symbols, preserve Windows PDBs, and keep symbol archives available to the release job. - Strip the packaged Linux `bwrap` binary before hashing it so the embedded digest matches the distributed bytes. ## Root cause The first symbol-artifact implementation enabled `CARGO_PROFILE_RELEASE_DEBUG=full`. In the June 2 release runs, macOS ARM primary builds reached the 90-minute timeout while still inside `Cargo build`. After the symbol changes were reverted, the same primary build completed in about 22 minutes. The archive step itself completed in tens of seconds when reached. Rust's `line-tables-only` debuginfo level preserves function names and source locations for symbolication without emitting the heavier variable and type information from full debuginfo. ## Validation - Ran `just fmt` from `codex-rs`. - Ran `just test-github-scripts` from the repository root: 23 tests passed. - Ran `bash -n` and `shellcheck` on `.github/scripts/archive-release-symbols-and-strip-binaries.sh`. - Parsed both modified workflows as YAML and ran `git diff --check`. - Built a macOS release smoke binary with `line-tables-only`, archived its dSYM through the restored script, stripped the production binary, and verified that `atos` resolves `symbol_smoke_function` to `main.rs:2`. - Ran Linux archive-script control-flow coverage with stubbed `objcopy` and `strip` commands. - Ran Windows PDB archive staging coverage and verified underscore-emitted Rust PDB names are staged under shipped hyphenated binary names. ## Follow-up The release workflow only runs for tags or manual dispatches, so CI cannot dry-run the full release matrix on this PR. The next release run will verify runner time and memory behavior under `line-tables-only`. --- ...hive-release-symbols-and-strip-binaries.sh | 119 ++++++++++++++++++ .github/workflows/rust-release-windows.yml | 33 ++++- .github/workflows/rust-release.yml | 39 +++++- codex-rs/Cargo.toml | 7 +- 4 files changed, 192 insertions(+), 6 deletions(-) create mode 100755 .github/scripts/archive-release-symbols-and-strip-binaries.sh diff --git a/.github/scripts/archive-release-symbols-and-strip-binaries.sh b/.github/scripts/archive-release-symbols-and-strip-binaries.sh new file mode 100755 index 000000000..3e5894bb9 --- /dev/null +++ b/.github/scripts/archive-release-symbols-and-strip-binaries.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: archive-release-symbols-and-strip-binaries.sh \ + --target \ + --artifact-name \ + --release-dir \ + --archive-dir \ + --binaries "" +EOF +} + +target="" +artifact_name="" +release_dir="" +archive_dir="" +binaries="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --target) + target="${2:?--target requires a value}" + shift 2 + ;; + --artifact-name) + artifact_name="${2:?--artifact-name requires a value}" + shift 2 + ;; + --release-dir) + release_dir="${2:?--release-dir requires a value}" + shift 2 + ;; + --archive-dir) + archive_dir="${2:?--archive-dir requires a value}" + shift 2 + ;; + --binaries) + binaries="${2:?--binaries requires a value}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unexpected argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "$target" || -z "$artifact_name" || -z "$release_dir" || -z "$archive_dir" || -z "$binaries" ]]; then + usage >&2 + exit 1 +fi + +symbols_root="${RUNNER_TEMP:-/tmp}/codex-symbols-${artifact_name}" +symbols_dir="${symbols_root}/codex-symbols-${artifact_name}" +archive_path="${archive_dir%/}/codex-symbols-${artifact_name}.tar.gz" +rm -rf "$symbols_root" +mkdir -p "$symbols_dir" "$archive_dir" +read -r -a binary_names <<< "$binaries" + +case "$target" in + *apple-darwin) + for binary in "${binary_names[@]}"; do + binary_path="${release_dir%/}/${binary}" + dsym_path="${binary_path}.dSYM" + if [[ ! -f "$binary_path" ]]; then + echo "Binary $binary_path not found" >&2 + exit 1 + fi + if [[ ! -d "$dsym_path" ]]; then + echo "dSYM $dsym_path not found" >&2 + exit 1 + fi + + cp -RL "$dsym_path" "${symbols_dir}/${binary}.dSYM" + strip -S -x "$binary_path" + done + ;; + *linux*) + objcopy_bin="${OBJCOPY:-objcopy}" + strip_bin="${STRIP:-strip}" + for binary in "${binary_names[@]}"; do + binary_path="${release_dir%/}/${binary}" + debug_path="${symbols_dir}/${binary}.debug" + if [[ ! -f "$binary_path" ]]; then + echo "Binary $binary_path not found" >&2 + exit 1 + fi + + "$objcopy_bin" --only-keep-debug "$binary_path" "$debug_path" + "$strip_bin" --strip-debug --strip-unneeded "$binary_path" + "$objcopy_bin" --add-gnu-debuglink="$debug_path" "$binary_path" + done + ;; + *windows*) + for binary in "${binary_names[@]}"; do + pdb_path="${release_dir%/}/${binary}.pdb" + if [[ ! -f "$pdb_path" ]]; then + echo "PDB $pdb_path not found" >&2 + exit 1 + fi + + cp "$pdb_path" "${symbols_dir}/${binary}.pdb" + done + ;; + *) + echo "No symbols packaging support for target: $target" >&2 + exit 1 + ;; +esac + +rm -f "$archive_path" +tar -C "$symbols_root" -czf "$archive_path" "codex-symbols-${artifact_name}" diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index 14d38b9db..67c5c63ac 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -112,10 +112,22 @@ jobs: - name: Stage Windows binaries shell: bash run: | - output_dir="target/${{ matrix.target }}/release/staged-${{ matrix.bundle }}" + release_dir="target/${{ matrix.target }}/release" + output_dir="$release_dir/staged-${{ matrix.bundle }}" mkdir -p "$output_dir" for binary in ${{ matrix.binaries }}; do - cp "target/${{ matrix.target }}/release/${binary}.exe" "$output_dir/${binary}.exe" + pdb_name="${binary//-/_}" + pdb_path="$release_dir/${pdb_name}.pdb" + if [[ ! -f "$pdb_path" ]]; then + pdb_path="$release_dir/${binary}.pdb" + fi + if [[ ! -f "$pdb_path" ]]; then + echo "PDB for $binary not found at $release_dir/${pdb_name}.pdb or $release_dir/${binary}.pdb" >&2 + exit 1 + fi + + cp "$release_dir/${binary}.exe" "$output_dir/${binary}.exe" + cp "$pdb_path" "$output_dir/${binary}.pdb" done - name: Upload Windows binaries @@ -201,6 +213,23 @@ jobs: account-name: ${{ secrets.AZURE_ARTIFACT_SIGNING_ACCOUNT_NAME }} certificate-profile-name: ${{ secrets.AZURE_ARTIFACT_SIGNING_CERTIFICATE_PROFILE_NAME }} + - name: Build symbols archive + shell: bash + run: | + bash "${GITHUB_WORKSPACE}/.github/scripts/archive-release-symbols-and-strip-binaries.sh" \ + --target "${{ matrix.target }}" \ + --artifact-name "${{ matrix.target }}" \ + --release-dir "target/${{ matrix.target }}/release" \ + --archive-dir "symbols-dist/${{ matrix.target }}" \ + --binaries "${WINDOWS_BINARIES}" + + - name: Upload symbols archive + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: ${{ matrix.target }}-symbols + path: codex-rs/symbols-dist/${{ matrix.target }}/* + if-no-files-found: error + - name: Stage artifacts shell: bash run: | diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index f95d1216f..5dbf1b77e 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -64,6 +64,8 @@ jobs: run: working-directory: codex-rs env: + # macOS release packages archive packed dSYM bundles before stripping. + CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO: ${{ contains(matrix.target, 'apple-darwin') && 'packed' || 'off' }} # Use the git CLI instead of Cargo's libgit2 path for git dependencies. # macOS release runners have intermittently failed to fetch nested # submodules through SecureTransport/libgit2, especially libwebrtc's @@ -163,7 +165,7 @@ jobs: run: | set -euo pipefail sudo apt-get update -y - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends binutils pkg-config libcap-dev - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 with: targets: ${{ matrix.target }} @@ -222,6 +224,10 @@ jobs: exit 1 fi + # Codex embeds this digest at build time and verifies the bundled + # bwrap resource before use. Strip bwrap before hashing so the digest + # covers the exact bytes that the release packages. + strip --strip-debug --strip-unneeded "$bwrap_path" digest="$(sha256sum "$bwrap_path" | awk '{print $1}')" echo "CODEX_BWRAP_SHA256=${digest}" >> "$GITHUB_ENV" echo "Built bwrap ${bwrap_path} with sha256:${digest}" @@ -235,6 +241,11 @@ jobs: fi build_args=() for binary in ${{ matrix.binaries }}; do + # bwrap was built, finalized, and hashed before this build so + # Codex can embed the digest of the bytes that will be packaged. + if [[ "$binary" == "bwrap" ]]; then + continue + fi build_args+=(--bin "$binary") done cargo build --target "$target" --release --timings "${build_args[@]}" @@ -246,6 +257,32 @@ jobs: path: codex-rs/target/**/cargo-timings/cargo-timing.html if-no-files-found: warn + - name: Build symbols archive and strip binaries + shell: bash + run: | + binaries=() + for binary in ${{ matrix.binaries }}; do + # bwrap is already stripped before hashing. Its symbols are not + # useful enough to justify a separate pre-Codex symbols pass. + if [[ "$binary" == "bwrap" ]]; then + continue + fi + binaries+=("$binary") + done + bash "${GITHUB_WORKSPACE}/.github/scripts/archive-release-symbols-and-strip-binaries.sh" \ + --target "${{ matrix.target }}" \ + --artifact-name "${{ matrix.artifact_name }}" \ + --release-dir "target/${{ matrix.target }}/release" \ + --archive-dir "symbols-dist/${{ matrix.artifact_name }}" \ + --binaries "${binaries[*]}" + + - name: Upload symbols archive + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: ${{ matrix.artifact_name }}-symbols + path: codex-rs/symbols-dist/${{ matrix.artifact_name }}/* + if-no-files-found: error + - if: ${{ runner.os == 'macOS' }} name: Stage unsigned macOS artifacts shell: bash diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 0e3aba328..3bb23ba80 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -501,10 +501,11 @@ strip = "symbols" [profile.release] lto = "thin" +debug = "line-tables-only" split-debuginfo = "off" -# Because we bundle some of these executables with the TypeScript CLI, we -# remove everything to make the binary as small as possible. -strip = "symbols" +# Keep release binaries symbolicateable until packaging has archived the +# sidecar symbols and stripped the binaries. +strip = false # See https://github.com/openai/codex/issues/1411 for details. codegen-units = 1