From 75a08def9808ed5b77b458d09c2fe99f2b52c433 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:49:54 -0700 Subject: [PATCH] [codex] Publish release symbol artifacts (#25649) ## Why Production Codex binaries are stripped for distribution, which leaves crashes and samples from released builds without the symbols needed for useful stack traces. Publish symbols as separate release assets so production artifacts stay small while released builds remain symbolicateable. ## What changed - Add `.github/scripts/archive-release-symbols-and-strip-binaries.sh` to package platform-native symbols into `codex-symbols-.tar.gz` assets while stripping the corresponding Unix binaries before signing. - Build release binaries with full debug information before producing distribution artifacts. - Publish macOS `.dSYM` bundles, Linux `.debug` files with `.gnu_debuglink`, and Windows `.pdb` files. - Strip Linux `bwrap` before computing its packaged-resource digest, but intentionally omit `bwrap` from symbol archives. - Preserve symbols artifacts in the unsigned macOS promotion flow. ## Verification - Ran `shellcheck` and `bash -n` on `.github/scripts/archive-release-symbols-and-strip-binaries.sh`. - Parsed the modified workflow YAML files and ran `git diff --check`. - Built a macOS release smoke binary and verified that the archived `.dSYM` contains DWARF application source information and has the same UUID as the stripped production binary. - Built Linux smoke binaries and verified that the symbol archive contains `codex.debug`, excludes `bwrap.debug`, leaves the expected `.gnu_debuglink` in `codex`, and does not mutate the separately stripped `bwrap` digest. - Staged a Windows smoke archive and verified that it contains the expected `.pdb` file. --- ...hive-release-symbols-and-strip-binaries.sh | 119 ++++++++++++++++++ .github/workflows/rust-release-windows.yml | 20 +++ .github/workflows/rust-release.yml | 41 +++++- 3 files changed, 179 insertions(+), 1 deletion(-) create mode 100755 .github/scripts/archive-release-symbols-and-strip-binaries.sh diff --git a/.github/scripts/archive-release-symbols-and-strip-binaries.sh b/.github/scripts/archive-release-symbols-and-strip-binaries.sh new file mode 100755 index 000000000..3e5894bb9 --- /dev/null +++ b/.github/scripts/archive-release-symbols-and-strip-binaries.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: archive-release-symbols-and-strip-binaries.sh \ + --target \ + --artifact-name \ + --release-dir \ + --archive-dir \ + --binaries "" +EOF +} + +target="" +artifact_name="" +release_dir="" +archive_dir="" +binaries="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --target) + target="${2:?--target requires a value}" + shift 2 + ;; + --artifact-name) + artifact_name="${2:?--artifact-name requires a value}" + shift 2 + ;; + --release-dir) + release_dir="${2:?--release-dir requires a value}" + shift 2 + ;; + --archive-dir) + archive_dir="${2:?--archive-dir requires a value}" + shift 2 + ;; + --binaries) + binaries="${2:?--binaries requires a value}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unexpected argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "$target" || -z "$artifact_name" || -z "$release_dir" || -z "$archive_dir" || -z "$binaries" ]]; then + usage >&2 + exit 1 +fi + +symbols_root="${RUNNER_TEMP:-/tmp}/codex-symbols-${artifact_name}" +symbols_dir="${symbols_root}/codex-symbols-${artifact_name}" +archive_path="${archive_dir%/}/codex-symbols-${artifact_name}.tar.gz" +rm -rf "$symbols_root" +mkdir -p "$symbols_dir" "$archive_dir" +read -r -a binary_names <<< "$binaries" + +case "$target" in + *apple-darwin) + for binary in "${binary_names[@]}"; do + binary_path="${release_dir%/}/${binary}" + dsym_path="${binary_path}.dSYM" + if [[ ! -f "$binary_path" ]]; then + echo "Binary $binary_path not found" >&2 + exit 1 + fi + if [[ ! -d "$dsym_path" ]]; then + echo "dSYM $dsym_path not found" >&2 + exit 1 + fi + + cp -RL "$dsym_path" "${symbols_dir}/${binary}.dSYM" + strip -S -x "$binary_path" + done + ;; + *linux*) + objcopy_bin="${OBJCOPY:-objcopy}" + strip_bin="${STRIP:-strip}" + for binary in "${binary_names[@]}"; do + binary_path="${release_dir%/}/${binary}" + debug_path="${symbols_dir}/${binary}.debug" + if [[ ! -f "$binary_path" ]]; then + echo "Binary $binary_path not found" >&2 + exit 1 + fi + + "$objcopy_bin" --only-keep-debug "$binary_path" "$debug_path" + "$strip_bin" --strip-debug --strip-unneeded "$binary_path" + "$objcopy_bin" --add-gnu-debuglink="$debug_path" "$binary_path" + done + ;; + *windows*) + for binary in "${binary_names[@]}"; do + pdb_path="${release_dir%/}/${binary}.pdb" + if [[ ! -f "$pdb_path" ]]; then + echo "PDB $pdb_path not found" >&2 + exit 1 + fi + + cp "$pdb_path" "${symbols_dir}/${binary}.pdb" + done + ;; + *) + echo "No symbols packaging support for target: $target" >&2 + exit 1 + ;; +esac + +rm -f "$archive_path" +tar -C "$symbols_root" -czf "$archive_path" "codex-symbols-${artifact_name}" diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index 3b4d3cfeb..91887cbe0 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -34,6 +34,8 @@ jobs: working-directory: codex-rs env: CARGO_PROFILE_RELEASE_LTO: ${{ inputs.release-lto }} + CARGO_PROFILE_RELEASE_DEBUG: full + CARGO_PROFILE_RELEASE_STRIP: "false" strategy: fail-fast: false @@ -131,6 +133,7 @@ jobs: mkdir -p "$output_dir" for binary in ${{ matrix.binaries }}; do cp "target/${{ matrix.target }}/release/${binary}.exe" "$output_dir/${binary}.exe" + cp "target/${{ matrix.target }}/release/${binary}.pdb" "$output_dir/${binary}.pdb" done - name: Upload Windows binaries @@ -213,6 +216,23 @@ jobs: account-name: ${{ secrets.AZURE_TRUSTED_SIGNING_ACCOUNT_NAME }} certificate-profile-name: ${{ secrets.AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME }} + - name: Build symbols archive + shell: bash + run: | + bash "${GITHUB_WORKSPACE}/.github/scripts/archive-release-symbols-and-strip-binaries.sh" \ + --target "${{ matrix.target }}" \ + --artifact-name "${{ matrix.target }}" \ + --release-dir "target/${{ matrix.target }}/release" \ + --archive-dir "symbols-dist/${{ matrix.target }}" \ + --binaries "${WINDOWS_BINARIES}" + + - name: Upload symbols archive + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: ${{ matrix.target }}-symbols + path: codex-rs/symbols-dist/${{ matrix.target }}/* + if-no-files-found: error + - name: Stage artifacts shell: bash run: | diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index db72ab830..364fe7890 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -149,6 +149,9 @@ jobs: # 2026-03-04: temporarily change releases to use thin LTO because # Ubuntu ARM is timing out at 60 minutes. CARGO_PROFILE_RELEASE_LTO: ${{ contains(github.ref_name, '-alpha') && 'thin' || 'thin' }} + CARGO_PROFILE_RELEASE_DEBUG: full + CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO: ${{ contains(matrix.target, 'apple-darwin') && 'packed' || 'off' }} + CARGO_PROFILE_RELEASE_STRIP: "false" # Use the git CLI instead of Cargo's libgit2 path for git dependencies. # macOS release runners have intermittently failed to fetch nested # submodules through SecureTransport/libgit2, especially libwebrtc's @@ -249,7 +252,7 @@ jobs: run: | set -euo pipefail sudo apt-get update -y - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends binutils pkg-config libcap-dev - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 with: targets: ${{ matrix.target }} @@ -308,6 +311,10 @@ jobs: exit 1 fi + # Codex embeds this digest at build time and verifies the bundled + # bwrap resource before use. Strip bwrap before hashing so the digest + # covers the exact bytes that the release packages. + strip --strip-debug --strip-unneeded "$bwrap_path" digest="$(sha256sum "$bwrap_path" | awk '{print $1}')" echo "CODEX_BWRAP_SHA256=${digest}" >> "$GITHUB_ENV" echo "Built bwrap ${bwrap_path} with sha256:${digest}" @@ -321,6 +328,11 @@ jobs: fi build_args=() for binary in ${{ matrix.binaries }}; do + # bwrap was built, finalized, and hashed before this build so + # Codex can embed the digest of the bytes that will be packaged. + if [[ "$binary" == "bwrap" ]]; then + continue + fi build_args+=(--bin "$binary") done echo "CARGO_PROFILE_RELEASE_LTO: ${CARGO_PROFILE_RELEASE_LTO}" @@ -333,6 +345,32 @@ jobs: path: codex-rs/target/**/cargo-timings/cargo-timing.html if-no-files-found: warn + - name: Build symbols archive and strip binaries + shell: bash + run: | + binaries=() + for binary in ${{ matrix.binaries }}; do + # bwrap is already stripped before hashing. Its symbols are not + # useful enough to justify a separate pre-Codex symbols pass. + if [[ "$binary" == "bwrap" ]]; then + continue + fi + binaries+=("$binary") + done + bash "${GITHUB_WORKSPACE}/.github/scripts/archive-release-symbols-and-strip-binaries.sh" \ + --target "${{ matrix.target }}" \ + --artifact-name "${{ matrix.artifact_name }}" \ + --release-dir "target/${{ matrix.target }}/release" \ + --archive-dir "symbols-dist/${{ matrix.artifact_name }}" \ + --binaries "${binaries[*]}" + + - name: Upload symbols archive + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: ${{ matrix.artifact_name }}-symbols + path: codex-rs/symbols-dist/${{ matrix.artifact_name }}/* + if-no-files-found: error + - if: ${{ runner.os == 'macOS' && env.SIGN_MACOS != 'true' }} name: Stage unsigned macOS artifacts shell: bash @@ -1031,6 +1069,7 @@ jobs: run: | find dist -mindepth 1 -maxdepth 1 -type d \ ! -name '*-apple-darwin*-unsigned' \ + ! -name '*-symbols' \ ! -name 'aarch64-unknown-linux-musl' \ ! -name 'aarch64-unknown-linux-musl-app-server' \ ! -name 'x86_64-unknown-linux-musl' \