V8 Bazel Build (#15021)

Alternative approach, we use rusty_v8 for all platforms that its
predefined, but lets build from source a musl v8 version with bazel for
x86 and aarch64 only. We would need to release this on github and then
use the release.

.github/scripts/rusty_v8_bazel.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import tomllib
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+MUSL_RUNTIME_ARCHIVE_LABELS = [
+    "@llvm//runtimes/libcxx:libcxx.static",
+    "@llvm//runtimes/libcxx:libcxxabi.static",
+]
+LLVM_AR_LABEL = "@llvm//tools:llvm-ar"
+LLVM_RANLIB_LABEL = "@llvm//tools:llvm-ranlib"
+
+
+def bazel_execroot() -> Path:
+    result = subprocess.run(
+        ["bazel", "info", "execution_root"],
+        cwd=ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return Path(result.stdout.strip())
+
+
+def bazel_output_base() -> Path:
+    result = subprocess.run(
+        ["bazel", "info", "output_base"],
+        cwd=ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return Path(result.stdout.strip())
+
+
+def bazel_output_path(path: str) -> Path:
+    if path.startswith("external/"):
+        return bazel_output_base() / path
+    return bazel_execroot() / path
+
+
+def bazel_output_files(
+    platform: str,
+    labels: list[str],
+    compilation_mode: str = "fastbuild",
+) -> list[Path]:
+    expression = "set(" + " ".join(labels) + ")"
+    result = subprocess.run(
+        [
+            "bazel",
+            "cquery",
+            "-c",
+            compilation_mode,
+            f"--platforms=@llvm//platforms:{platform}",
+            "--output=files",
+            expression,
+        ],
+        cwd=ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return [bazel_output_path(line.strip()) for line in result.stdout.splitlines() if line.strip()]
+
+
+def bazel_build(
+    platform: str,
+    labels: list[str],
+    compilation_mode: str = "fastbuild",
+) -> None:
+    subprocess.run(
+        [
+            "bazel",
+            "build",
+            "-c",
+            compilation_mode,
+            f"--platforms=@llvm//platforms:{platform}",
+            *labels,
+        ],
+        cwd=ROOT,
+        check=True,
+    )
+
+
+def ensure_bazel_output_files(
+    platform: str,
+    labels: list[str],
+    compilation_mode: str = "fastbuild",
+) -> list[Path]:
+    outputs = bazel_output_files(platform, labels, compilation_mode)
+    if all(path.exists() for path in outputs):
+        return outputs
+
+    bazel_build(platform, labels, compilation_mode)
+    outputs = bazel_output_files(platform, labels, compilation_mode)
+    missing = [str(path) for path in outputs if not path.exists()]
+    if missing:
+        raise SystemExit(f"missing built outputs for {labels}: {missing}")
+    return outputs
+
+
+def release_pair_label(target: str) -> str:
+    target_suffix = target.replace("-", "_")
+    return f"//third_party/v8:rusty_v8_release_pair_{target_suffix}"
+
+
+def resolved_v8_crate_version() -> str:
+    cargo_lock = tomllib.loads((ROOT / "codex-rs" / "Cargo.lock").read_text())
+    versions = sorted(
+        {
+            package["version"]
+            for package in cargo_lock["package"]
+            if package["name"] == "v8"
+        }
+    )
+    if len(versions) == 1:
+        return versions[0]
+    if len(versions) > 1:
+        raise SystemExit(f"expected exactly one resolved v8 version, found: {versions}")
+
+    module_bazel = (ROOT / "MODULE.bazel").read_text()
+    matches = sorted(
+        set(
+            re.findall(
+                r'https://static\.crates\.io/crates/v8/v8-([0-9]+\.[0-9]+\.[0-9]+)\.crate',
+                module_bazel,
+            )
+        )
+    )
+    if len(matches) != 1:
+        raise SystemExit(
+            "expected exactly one pinned v8 crate version in MODULE.bazel, "
+            f"found: {matches}"
+        )
+    return matches[0]
+
+
+def staged_archive_name(target: str, source_path: Path) -> str:
+    if source_path.suffix == ".lib":
+        return f"rusty_v8_release_{target}.lib.gz"
+    return f"librusty_v8_release_{target}.a.gz"
+
+
+def is_musl_archive_target(target: str, source_path: Path) -> bool:
+    return target.endswith("-unknown-linux-musl") and source_path.suffix == ".a"
+
+
+def single_bazel_output_file(
+    platform: str,
+    label: str,
+    compilation_mode: str = "fastbuild",
+) -> Path:
+    outputs = ensure_bazel_output_files(platform, [label], compilation_mode)
+    if len(outputs) != 1:
+        raise SystemExit(f"expected exactly one output for {label}, found {outputs}")
+    return outputs[0]
+
+
+def merged_musl_archive(
+    platform: str,
+    lib_path: Path,
+    compilation_mode: str = "fastbuild",
+) -> Path:
+    llvm_ar = single_bazel_output_file(platform, LLVM_AR_LABEL, compilation_mode)
+    llvm_ranlib = single_bazel_output_file(platform, LLVM_RANLIB_LABEL, compilation_mode)
+    runtime_archives = [
+        single_bazel_output_file(platform, label, compilation_mode)
+        for label in MUSL_RUNTIME_ARCHIVE_LABELS
+    ]
+
+    temp_dir = Path(tempfile.mkdtemp(prefix="rusty-v8-musl-stage-"))
+    merged_archive = temp_dir / lib_path.name
+    merge_commands = "\n".join(
+        [
+            f"create {merged_archive}",
+            f"addlib {lib_path}",
+            *[f"addlib {archive}" for archive in runtime_archives],
+            "save",
+            "end",
+        ]
+    )
+    subprocess.run(
+        [str(llvm_ar), "-M"],
+        cwd=ROOT,
+        check=True,
+        input=merge_commands,
+        text=True,
+    )
+    subprocess.run([str(llvm_ranlib), str(merged_archive)], cwd=ROOT, check=True)
+    return merged_archive
+
+
+def stage_release_pair(
+    platform: str,
+    target: str,
+    output_dir: Path,
+    compilation_mode: str = "fastbuild",
+) -> None:
+    outputs = ensure_bazel_output_files(
+        platform,
+        [release_pair_label(target)],
+        compilation_mode,
+    )
+
+    try:
+        lib_path = next(path for path in outputs if path.suffix in {".a", ".lib"})
+    except StopIteration as exc:
+        raise SystemExit(f"missing static library output for {target}") from exc
+
+    try:
+        binding_path = next(path for path in outputs if path.suffix == ".rs")
+    except StopIteration as exc:
+        raise SystemExit(f"missing Rust binding output for {target}") from exc
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    staged_library = output_dir / staged_archive_name(target, lib_path)
+    staged_binding = output_dir / f"src_binding_release_{target}.rs"
+    source_archive = (
+        merged_musl_archive(platform, lib_path, compilation_mode)
+        if is_musl_archive_target(target, lib_path)
+        else lib_path
+    )
+
+    with source_archive.open("rb") as src, staged_library.open("wb") as dst:
+        with gzip.GzipFile(
+            filename="",
+            mode="wb",
+            fileobj=dst,
+            compresslevel=6,
+            mtime=0,
+        ) as gz:
+            shutil.copyfileobj(src, gz)
+
+    shutil.copyfile(binding_path, staged_binding)
+
+    print(staged_library)
+    print(staged_binding)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    stage_release_pair_parser = subparsers.add_parser("stage-release-pair")
+    stage_release_pair_parser.add_argument("--platform", required=True)
+    stage_release_pair_parser.add_argument("--target", required=True)
+    stage_release_pair_parser.add_argument("--output-dir", required=True)
+    stage_release_pair_parser.add_argument(
+        "--compilation-mode",
+        default="fastbuild",
+        choices=["fastbuild", "opt", "dbg"],
+    )
+
+    subparsers.add_parser("resolved-v8-crate-version")
+
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    if args.command == "stage-release-pair":
+        stage_release_pair(
+            platform=args.platform,
+            target=args.target,
+            output_dir=Path(args.output_dir),
+            compilation_mode=args.compilation_mode,
+        )
+        return 0
+    if args.command == "resolved-v8-crate-version":
+        print(resolved_v8_crate_version())
+        return 0
+    raise SystemExit(f"unsupported command: {args.command}")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/bazel.yml

@@ -156,7 +156,6 @@ jobs:
 
           bazel_args=(
             test
-            //...
             --test_verbose_timeout_warnings
             --build_metadata=REPO_URL=https://github.com/openai/codex.git
             --build_metadata=COMMIT_SHA=$(git rev-parse HEAD)
@@ -164,6 +163,13 @@ jobs:
             --build_metadata=VISIBILITY=PUBLIC
           )
 
+          bazel_targets=(
+            //...
+            # Keep V8 out of the ordinary Bazel CI path. Only the dedicated
+            # canary and release workflows should build `third_party/v8`.
+            -//third_party/v8:all
+          )
+
           if [[ "${RUNNER_OS:-}" != "Windows" ]]; then
             # Bazel test sandboxes on macOS may resolve an older Homebrew `node`
             # before the `actions/setup-node` runtime on PATH.
@@ -183,6 +189,8 @@ jobs:
               --bazelrc=.github/workflows/ci.bazelrc \
               "${bazel_args[@]}" \
               "--remote_header=x-buildbuddy-api-key=$BUILDBUDDY_API_KEY" \
+              -- \
+              "${bazel_targets[@]}" \
               2>&1 | tee "$bazel_console_log"
             bazel_status=${PIPESTATUS[0]}
             set -e
@@ -210,6 +218,8 @@ jobs:
               "${bazel_args[@]}" \
               --remote_cache= \
               --remote_executor= \
+              -- \
+              "${bazel_targets[@]}" \
               2>&1 | tee "$bazel_console_log"
             bazel_status=${PIPESTATUS[0]}
             set -e

.github/workflows/rusty-v8-release.yml

@@ -0,0 +1,188 @@
+name: rusty-v8-release
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: Optional release tag. Defaults to rusty-v8-v<resolved_v8_version>.
+        required: false
+        type: string
+      publish:
+        description: Publish the staged musl artifacts to a GitHub release.
+        required: false
+        default: true
+        type: boolean
+
+concurrency:
+  group: ${{ github.workflow }}::${{ inputs.release_tag || github.run_id }}
+  cancel-in-progress: false
+
+jobs:
+  metadata:
+    runs-on: ubuntu-latest
+    outputs:
+      release_tag: ${{ steps.release_tag.outputs.release_tag }}
+      v8_version: ${{ steps.v8_version.outputs.version }}
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Resolve exact v8 crate version
+        id: v8_version
+        shell: bash
+        run: |
+          set -euo pipefail
+          version="$(python3 .github/scripts/rusty_v8_bazel.py resolved-v8-crate-version)"
+          echo "version=${version}" >> "$GITHUB_OUTPUT"
+
+      - name: Resolve release tag
+        id: release_tag
+        env:
+          RELEASE_TAG_INPUT: ${{ inputs.release_tag }}
+          V8_VERSION: ${{ steps.v8_version.outputs.version }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          release_tag="${RELEASE_TAG_INPUT}"
+          if [[ -z "${release_tag}" ]]; then
+            release_tag="rusty-v8-v${V8_VERSION}"
+          fi
+
+          echo "release_tag=${release_tag}" >> "$GITHUB_OUTPUT"
+
+  build:
+    name: Build ${{ matrix.target }}
+    needs: metadata
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
+      actions: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            platform: linux_amd64_musl
+            target: x86_64-unknown-linux-musl
+          - runner: ubuntu-24.04-arm
+            platform: linux_arm64_musl
+            target: aarch64-unknown-linux-musl
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Bazel
+        uses: bazelbuild/setup-bazelisk@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Build Bazel V8 release pair
+        env:
+          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
+          PLATFORM: ${{ matrix.platform }}
+          TARGET: ${{ matrix.target }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          target_suffix="${TARGET//-/_}"
+          pair_target="//third_party/v8:rusty_v8_release_pair_${target_suffix}"
+          extra_targets=()
+          if [[ "${TARGET}" == *-unknown-linux-musl ]]; then
+            extra_targets=(
+              "@llvm//runtimes/libcxx:libcxx.static"
+              "@llvm//runtimes/libcxx:libcxxabi.static"
+            )
+          fi
+
+          bazel_args=(
+            build
+            -c
+            opt
+            "--platforms=@llvm//platforms:${PLATFORM}"
+            "${pair_target}"
+            "${extra_targets[@]}"
+            --build_metadata=COMMIT_SHA=$(git rev-parse HEAD)
+          )
+
+          bazel \
+            --noexperimental_remote_repo_contents_cache \
+            --bazelrc=.github/workflows/v8-ci.bazelrc \
+            "${bazel_args[@]}" \
+            "--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}"
+
+      - name: Stage release pair
+        env:
+          PLATFORM: ${{ matrix.platform }}
+          TARGET: ${{ matrix.target }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          python3 .github/scripts/rusty_v8_bazel.py stage-release-pair \
+            --platform "${PLATFORM}" \
+            --target "${TARGET}" \
+            --compilation-mode opt \
+            --output-dir "dist/${TARGET}"
+
+      - name: Upload staged musl artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: rusty-v8-${{ needs.metadata.outputs.v8_version }}-${{ matrix.target }}
+          path: dist/${{ matrix.target }}/*
+
+  publish-release:
+    if: ${{ inputs.publish }}
+    needs:
+      - metadata
+      - build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      actions: read
+
+    steps:
+      - name: Ensure publishing from default branch
+        if: ${{ github.ref_name != github.event.repository.default_branch }}
+        env:
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          echo "Publishing is only allowed from ${DEFAULT_BRANCH}; current ref is ${GITHUB_REF_NAME}." >&2
+          exit 1
+
+      - name: Ensure release tag is new
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_TAG: ${{ needs.metadata.outputs.release_tag }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if gh release view "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" > /dev/null 2>&1; then
+            echo "Release tag ${RELEASE_TAG} already exists; musl artifact tags are immutable." >&2
+            exit 1
+          fi
+
+      - uses: actions/download-artifact@v8
+        with:
+          path: dist
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ needs.metadata.outputs.release_tag }}
+          name: ${{ needs.metadata.outputs.release_tag }}
+          files: dist/**
+          # Keep V8 artifact releases out of Codex's normal "latest release" channel.
+          prerelease: true

.github/workflows/v8-canary.yml

@@ -0,0 +1,132 @@
+name: v8-canary
+
+on:
+  pull_request:
+    paths:
+      - ".github/scripts/rusty_v8_bazel.py"
+      - ".github/workflows/rusty-v8-release.yml"
+      - ".github/workflows/v8-canary.yml"
+      - "MODULE.bazel"
+      - "MODULE.bazel.lock"
+      - "codex-rs/Cargo.toml"
+      - "patches/BUILD.bazel"
+      - "patches/v8_*.patch"
+      - "third_party/v8/**"
+  push:
+    branches:
+      - main
+    paths:
+      - ".github/scripts/rusty_v8_bazel.py"
+      - ".github/workflows/rusty-v8-release.yml"
+      - ".github/workflows/v8-canary.yml"
+      - "MODULE.bazel"
+      - "MODULE.bazel.lock"
+      - "codex-rs/Cargo.toml"
+      - "patches/BUILD.bazel"
+      - "patches/v8_*.patch"
+      - "third_party/v8/**"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}::${{ github.event.pull_request.number > 0 && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}
+  cancel-in-progress: ${{ github.ref_name != 'main' }}
+
+jobs:
+  metadata:
+    runs-on: ubuntu-latest
+    outputs:
+      v8_version: ${{ steps.v8_version.outputs.version }}
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Resolve exact v8 crate version
+        id: v8_version
+        shell: bash
+        run: |
+          set -euo pipefail
+          version="$(python3 .github/scripts/rusty_v8_bazel.py resolved-v8-crate-version)"
+          echo "version=${version}" >> "$GITHUB_OUTPUT"
+
+  build:
+    name: Build ${{ matrix.target }}
+    needs: metadata
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
+      actions: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            platform: linux_amd64_musl
+            target: x86_64-unknown-linux-musl
+          - runner: ubuntu-24.04-arm
+            platform: linux_arm64_musl
+            target: aarch64-unknown-linux-musl
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Bazel
+        uses: bazelbuild/setup-bazelisk@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Build Bazel V8 release pair
+        env:
+          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
+          PLATFORM: ${{ matrix.platform }}
+          TARGET: ${{ matrix.target }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          target_suffix="${TARGET//-/_}"
+          pair_target="//third_party/v8:rusty_v8_release_pair_${target_suffix}"
+          extra_targets=(
+            "@llvm//runtimes/libcxx:libcxx.static"
+            "@llvm//runtimes/libcxx:libcxxabi.static"
+          )
+
+          bazel_args=(
+            build
+            "--platforms=@llvm//platforms:${PLATFORM}"
+            "${pair_target}"
+            "${extra_targets[@]}"
+            --build_metadata=COMMIT_SHA=$(git rev-parse HEAD)
+          )
+
+          bazel \
+            --noexperimental_remote_repo_contents_cache \
+            --bazelrc=.github/workflows/v8-ci.bazelrc \
+            "${bazel_args[@]}" \
+            "--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}"
+
+      - name: Stage release pair
+        env:
+          PLATFORM: ${{ matrix.platform }}
+          TARGET: ${{ matrix.target }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          python3 .github/scripts/rusty_v8_bazel.py stage-release-pair \
+            --platform "${PLATFORM}" \
+            --target "${TARGET}" \
+            --output-dir "dist/${TARGET}"
+
+      - name: Upload staged musl artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: v8-canary-${{ needs.metadata.outputs.v8_version }}-${{ matrix.target }}
+          path: dist/${{ matrix.target }}/*

.github/workflows/v8-ci.bazelrc

@@ -0,0 +1,5 @@
+import %workspace%/.github/workflows/ci.bazelrc
+
+common --build_metadata=REPO_URL=https://github.com/openai/codex.git
+common --build_metadata=ROLE=CI
+common --build_metadata=VISIBILITY=PUBLIC