mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
Route Bazel CI through shared BuildBuddy remote config wrapper (#25156)
## Why Bazel remote configuration was selected in several CI scripts and workflow steps. That made the BuildBuddy tenant policy easy to duplicate and harder to audit, especially for fork pull requests that must not use the OpenAI tenant. This builds on [sluongng/buildbuddy-ci-host-routing](https://github.com/openai/codex/compare/main...sluongng:codex:sluongng/buildbuddy-ci-host-routing) and consolidates the policy in one place. ## What to do if this breaks you See `codex-rs/docs/bazel.md` for details. TLDR: 1. make a BuildBuddy API key and put it in `~/.bazelrc` 2. if you're an OpenAI employee, add `common --config=buildbuddy-openai-rbe` to `user.bazelrc` in the repo root Run `just bazel-test` to ensure it works. Note that `just bazel-remote-test` no longer exists, you need to select a remote configuration as documented to use RBE. ## What changed - Add `.github/scripts/run_bazel_with_buildbuddy.py` as the shared Bazel wrapper and Python library. It selects the OpenAI host only for trusted upstream GitHub Actions runs, routes keyed fork runs to the generic host, and falls back to local Bazel execution when no key is available. - Move endpoint selection into explicit `.bazelrc` configurations and update Bazel CI, query helpers, and `rusty_v8` staging to use the shared policy. Loading-phase target-discovery queries remain local. - Add wrapper and `rusty_v8` unit coverage, plus `just test-scripts` for the `.github/scripts` Python tests. - Document local Bazel usage, `user.bazelrc` setup, BuildBuddy configurations, and CI behavior in `codex-rs/docs/bazel.md`. ## Validation - `just test-scripts` - `bash -n .github/scripts/run-bazel-ci.sh .github/scripts/run-bazel-query-ci.sh .github/scripts/run-argument-comment-lint-bazel.sh scripts/list-bazel-clippy-targets.sh` - `python3 -m py_compile .github/scripts/run_bazel_with_buildbuddy.py .github/scripts/test_run_bazel_with_buildbuddy.py .github/scripts/test_rusty_v8_bazel.py .github/scripts/rusty_v8_bazel.py` - `ruff check .github/scripts/run_bazel_with_buildbuddy.py .github/scripts/test_run_bazel_with_buildbuddy.py .github/scripts/test_rusty_v8_bazel.py .github/scripts/rusty_v8_bazel.py`
This commit is contained in:
committed by
GitHub
Unverified
parent
859dbe2761
commit
ebb7980369
@@ -53,11 +53,20 @@ fi
|
||||
|
||||
run_bazel() {
|
||||
if [[ "${RUNNER_OS:-}" == "Windows" ]]; then
|
||||
MSYS2_ARG_CONV_EXCL='*' bazel "$@"
|
||||
MSYS2_ARG_CONV_EXCL='*' "$(dirname "${BASH_SOURCE[0]}")/run_bazel_with_buildbuddy.py" "$@"
|
||||
return
|
||||
fi
|
||||
|
||||
bazel "$@"
|
||||
"$(dirname "${BASH_SOURCE[0]}")/run_bazel_with_buildbuddy.py" "$@"
|
||||
}
|
||||
|
||||
run_bazel_with_startup_args() {
|
||||
if (( ${#bazel_startup_args[@]} > 0 )); then
|
||||
run_bazel "${bazel_startup_args[@]}" "$@"
|
||||
return
|
||||
fi
|
||||
|
||||
run_bazel "$@"
|
||||
}
|
||||
|
||||
ci_config=ci-linux
|
||||
@@ -77,23 +86,16 @@ esac
|
||||
print_bazel_test_log_tails() {
|
||||
local console_log="$1"
|
||||
local testlogs_dir
|
||||
local -a bazel_info_cmd=(bazel)
|
||||
|
||||
local -a bazel_info_args=(info)
|
||||
|
||||
if (( ${#bazel_startup_args[@]} > 0 )); then
|
||||
bazel_info_cmd+=("${bazel_startup_args[@]}")
|
||||
fi
|
||||
|
||||
# `bazel info` needs the same CI config as the failed test invocation so
|
||||
# platform-specific output roots match. On Windows, omitting `ci-windows`
|
||||
# would point at `local_windows-fastbuild` even when the test ran with the
|
||||
# MSVC host platform under `local_windows_msvc-fastbuild`.
|
||||
if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then
|
||||
bazel_info_args+=(
|
||||
"--config=${ci_config}"
|
||||
"--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}"
|
||||
)
|
||||
# `bazel info` needs the same CI config as the failed test invocation so
|
||||
# platform-specific output roots match. On Windows, omitting `ci-windows`
|
||||
# would point at `local_windows-fastbuild` even when the test ran with the
|
||||
# MSVC host platform under `local_windows_msvc-fastbuild`.
|
||||
bazel_info_args+=("--config=${ci_config}")
|
||||
fi
|
||||
|
||||
# Only pass flags that affect Bazel's output-root selection or repository
|
||||
# lookup. Test/build-only flags such as execution logs or remote download
|
||||
# mode can make `bazel info` fail, which would hide the real test log path.
|
||||
@@ -105,7 +107,7 @@ print_bazel_test_log_tails() {
|
||||
esac
|
||||
done
|
||||
|
||||
testlogs_dir="$(run_bazel "${bazel_info_cmd[@]:1}" \
|
||||
testlogs_dir="$(run_bazel_with_startup_args \
|
||||
--noexperimental_remote_repo_contents_cache \
|
||||
"${bazel_info_args[@]}" \
|
||||
bazel-testlogs 2>/dev/null || echo bazel-testlogs)"
|
||||
@@ -254,8 +256,9 @@ if [[ ${#bazel_args[@]} -eq 0 || ${#bazel_targets[@]} -eq 0 ]]; then
|
||||
fi
|
||||
|
||||
if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then
|
||||
# Fork PRs do not receive the BuildBuddy secret needed for the remote
|
||||
# cross-compile config. Preserve the previous local Windows build shape.
|
||||
# Windows cross-compilation depends on authenticated RBE. Preserve the local
|
||||
# Windows build shape when credentials are unavailable.
|
||||
ci_config=ci-windows
|
||||
windows_msvc_host_platform=1
|
||||
fi
|
||||
|
||||
@@ -297,9 +300,9 @@ if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -n "${BUI
|
||||
fi
|
||||
|
||||
if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then
|
||||
# The Windows cross-compile config depends on remote execution. Fork PRs do
|
||||
# not receive the BuildBuddy secret, so fall back to the existing local build
|
||||
# shape and keep its lower concurrency cap.
|
||||
# The Windows cross-compile config depends on authenticated remote
|
||||
# execution. When credentials are unavailable, keep the local build shape
|
||||
# and its lower concurrency cap.
|
||||
post_config_bazel_args+=(--jobs=8)
|
||||
fi
|
||||
|
||||
@@ -377,70 +380,31 @@ fi
|
||||
bazel_console_log="$(mktemp)"
|
||||
trap 'rm -f "$bazel_console_log"' EXIT
|
||||
|
||||
bazel_cmd=(bazel)
|
||||
if (( ${#bazel_startup_args[@]} > 0 )); then
|
||||
bazel_cmd+=("${bazel_startup_args[@]}")
|
||||
fi
|
||||
|
||||
bazel_run_args=(
|
||||
"${bazel_args[@]}"
|
||||
)
|
||||
if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then
|
||||
echo "BuildBuddy API key is available; using remote Bazel configuration."
|
||||
# Work around Bazel 9 remote repo contents cache / overlay materialization failures
|
||||
# seen in CI (for example "is not a symlink" or permission errors while
|
||||
# materializing external repos such as rules_perl). We still use BuildBuddy for
|
||||
# remote execution/cache; this only disables the startup-level repo contents cache.
|
||||
bazel_run_args=(
|
||||
"${bazel_args[@]}"
|
||||
"--config=${ci_config}"
|
||||
"--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}"
|
||||
)
|
||||
if (( ${#post_config_bazel_args[@]} > 0 )); then
|
||||
bazel_run_args+=("${post_config_bazel_args[@]}")
|
||||
fi
|
||||
set +e
|
||||
run_bazel "${bazel_cmd[@]:1}" \
|
||||
--noexperimental_remote_repo_contents_cache \
|
||||
"${bazel_run_args[@]}" \
|
||||
-- \
|
||||
"${bazel_targets[@]}" \
|
||||
2>&1 | tee "$bazel_console_log"
|
||||
bazel_status=${PIPESTATUS[0]}
|
||||
set -e
|
||||
bazel_run_args+=("--config=${ci_config}")
|
||||
else
|
||||
echo "BuildBuddy API key is not available; using local Bazel configuration."
|
||||
# Keep fork/community PRs on Bazel but disable remote services that are
|
||||
# configured in .bazelrc and require auth.
|
||||
#
|
||||
# Flag docs:
|
||||
# - Command-line reference: https://bazel.build/reference/command-line-reference
|
||||
# - Remote caching overview: https://bazel.build/remote/caching
|
||||
# - Remote execution overview: https://bazel.build/remote/rbe
|
||||
# - Build Event Protocol overview: https://bazel.build/remote/bep
|
||||
#
|
||||
# --noexperimental_remote_repo_contents_cache:
|
||||
# disable remote repo contents cache enabled in .bazelrc startup options.
|
||||
# https://bazel.build/reference/command-line-reference#startup_options-flag--experimental_remote_repo_contents_cache
|
||||
# --remote_cache= and --remote_executor=:
|
||||
# clear remote cache/execution endpoints configured in .bazelrc.
|
||||
# https://bazel.build/reference/command-line-reference#common_options-flag--remote_cache
|
||||
# https://bazel.build/reference/command-line-reference#common_options-flag--remote_executor
|
||||
bazel_run_args=(
|
||||
"${bazel_args[@]}"
|
||||
--remote_cache=
|
||||
--remote_executor=
|
||||
)
|
||||
if (( ${#post_config_bazel_args[@]} > 0 )); then
|
||||
bazel_run_args+=("${post_config_bazel_args[@]}")
|
||||
fi
|
||||
set +e
|
||||
run_bazel "${bazel_cmd[@]:1}" \
|
||||
--noexperimental_remote_repo_contents_cache \
|
||||
"${bazel_run_args[@]}" \
|
||||
-- \
|
||||
"${bazel_targets[@]}" \
|
||||
2>&1 | tee "$bazel_console_log"
|
||||
bazel_status=${PIPESTATUS[0]}
|
||||
set -e
|
||||
fi
|
||||
if (( ${#post_config_bazel_args[@]} > 0 )); then
|
||||
bazel_run_args+=("${post_config_bazel_args[@]}")
|
||||
fi
|
||||
set +e
|
||||
# Work around Bazel 9 remote repo contents cache / overlay materialization
|
||||
# failures seen in CI (for example "is not a symlink" or permission errors
|
||||
# while materializing external repos such as rules_perl). This only disables
|
||||
# the startup-level repo contents cache; keyed runs still use BuildBuddy.
|
||||
run_bazel_with_startup_args \
|
||||
--noexperimental_remote_repo_contents_cache \
|
||||
"${bazel_run_args[@]}" \
|
||||
-- \
|
||||
"${bazel_targets[@]}" \
|
||||
2>&1 | tee "$bazel_console_log"
|
||||
bazel_status=${PIPESTATUS[0]}
|
||||
set -e
|
||||
|
||||
if [[ ${bazel_status:-0} -ne 0 ]]; then
|
||||
if [[ $print_failed_bazel_action_summary -eq 1 ]]; then
|
||||
|
||||
@@ -2,48 +2,17 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Run Bazel queries with the same CI startup settings as the main build/test
|
||||
# invocation so target-discovery queries can reuse the same Bazel server.
|
||||
# Run target-discovery queries with the same startup settings as the main
|
||||
# build/test invocation so they can reuse the same Bazel server. Queries only
|
||||
# enumerate labels, so they intentionally do not select CI or remote configs.
|
||||
|
||||
query_args=()
|
||||
windows_cross_compile=0
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--windows-cross-compile)
|
||||
windows_cross_compile=1
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
query_args+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ $# -ne 1 ]]; then
|
||||
echo "Usage: $0 [--windows-cross-compile] [<bazel query args>...] -- <query expression>" >&2
|
||||
if [[ $# -lt 2 || "${@: -2:1}" != "--" ]]; then
|
||||
echo "Usage: $0 [<bazel query args>...] -- <query expression>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
query_expression="$1"
|
||||
|
||||
ci_config=ci-linux
|
||||
case "${RUNNER_OS:-}" in
|
||||
macOS)
|
||||
ci_config=ci-macos
|
||||
;;
|
||||
Windows)
|
||||
if [[ $windows_cross_compile -eq 1 ]]; then
|
||||
ci_config=ci-windows-cross
|
||||
else
|
||||
ci_config=ci-windows
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
query_args=("${@:1:$#-2}")
|
||||
query_expression="${@: -1}"
|
||||
|
||||
bazel_startup_args=()
|
||||
if [[ -n "${BAZEL_OUTPUT_USER_ROOT:-}" ]]; then
|
||||
@@ -60,12 +29,6 @@ run_bazel() {
|
||||
}
|
||||
|
||||
bazel_query_args=(--noexperimental_remote_repo_contents_cache query)
|
||||
if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then
|
||||
bazel_query_args+=(
|
||||
"--config=${ci_config}"
|
||||
"--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}"
|
||||
)
|
||||
fi
|
||||
|
||||
if [[ -n "${BAZEL_REPO_CONTENTS_CACHE:-}" ]]; then
|
||||
bazel_query_args+=("--repo_contents_cache=${BAZEL_REPO_CONTENTS_CACHE}")
|
||||
@@ -75,7 +38,10 @@ if [[ -n "${BAZEL_REPOSITORY_CACHE:-}" ]]; then
|
||||
bazel_query_args+=("--repository_cache=${BAZEL_REPOSITORY_CACHE}")
|
||||
fi
|
||||
|
||||
bazel_query_args+=("${query_args[@]}" "$query_expression")
|
||||
if (( ${#query_args[@]} > 0 )); then
|
||||
bazel_query_args+=("${query_args[@]}")
|
||||
fi
|
||||
bazel_query_args+=("$query_expression")
|
||||
|
||||
if (( ${#bazel_startup_args[@]} > 0 )); then
|
||||
run_bazel "${bazel_startup_args[@]}" "${bazel_query_args[@]}"
|
||||
|
||||
Executable
+142
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
OPENAI_REPOSITORY = "openai/codex"
|
||||
# Remote configurations select cache/BES/download endpoints. Their -rbe forms
|
||||
# also select the matching remote executor endpoint.
|
||||
GENERIC_REMOTE_CONFIG = "buildbuddy-generic"
|
||||
OPENAI_REMOTE_CONFIG = "buildbuddy-openai"
|
||||
# These CI configurations require remote build execution. The wrapper supplies
|
||||
# an RBE configuration, which also includes the common `remote` settings.
|
||||
REMOTE_EXECUTION_CONFIGS = {
|
||||
"--config=ci-linux",
|
||||
"--config=ci-macos",
|
||||
"--config=ci-v8",
|
||||
"--config=ci-windows-cross",
|
||||
}
|
||||
# Only authenticated workflow runs executing trusted upstream code may use the
|
||||
# OpenAI BuildBuddy host. A pull request event without proof that its head is
|
||||
# in the upstream repository fails closed to the generic host.
|
||||
def is_trusted_upstream_run(env: Mapping[str, str]) -> bool:
|
||||
# `GITHUB_REPOSITORY` is easy to set locally. Requiring GitHub's workflow
|
||||
# marker prevents a local command from opting itself into the OpenAI host.
|
||||
if (
|
||||
env.get("GITHUB_ACTIONS") != "true"
|
||||
or env.get("GITHUB_REPOSITORY") != OPENAI_REPOSITORY
|
||||
):
|
||||
return False
|
||||
# Non-PR workflow runs in `openai/codex` execute upstream refs, so they are
|
||||
# trusted. Fork code reaches these workflows only through pull requests.
|
||||
if env.get("GITHUB_EVENT_NAME") != "pull_request":
|
||||
return True
|
||||
|
||||
event_path = env.get("GITHUB_EVENT_PATH")
|
||||
if not event_path:
|
||||
return False
|
||||
try:
|
||||
event = json.loads(Path(event_path).read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return False
|
||||
|
||||
try:
|
||||
return event["pull_request"]["head"]["repo"]["fork"] is False
|
||||
except (KeyError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def uses_openai_host(env: Mapping[str, str]) -> bool:
|
||||
return bool(env.get("BUILDBUDDY_API_KEY")) and is_trusted_upstream_run(env)
|
||||
|
||||
|
||||
def uses_remote_execution(args: Sequence[str]) -> bool:
|
||||
try:
|
||||
separator_idx = args.index("--")
|
||||
except ValueError:
|
||||
separator_idx = len(args)
|
||||
return any(arg in REMOTE_EXECUTION_CONFIGS for arg in args[:separator_idx])
|
||||
|
||||
|
||||
def remote_config(args: Sequence[str], env: Mapping[str, str]) -> str | None:
|
||||
if not env.get("BUILDBUDDY_API_KEY"):
|
||||
return None
|
||||
|
||||
config = OPENAI_REMOTE_CONFIG if uses_openai_host(env) else GENERIC_REMOTE_CONFIG
|
||||
if uses_remote_execution(args):
|
||||
config += "-rbe"
|
||||
return config
|
||||
|
||||
|
||||
def bazel_args_without_remote_execution(args: Sequence[str]) -> list[str]:
|
||||
# Remote CI configs require BuildBuddy credentials. Removing them preserves
|
||||
# the local fallback used for fork pull requests.
|
||||
try:
|
||||
separator_idx = args.index("--")
|
||||
except ValueError:
|
||||
separator_idx = len(args)
|
||||
return [
|
||||
*(arg for arg in args[:separator_idx] if arg not in REMOTE_EXECUTION_CONFIGS),
|
||||
*args[separator_idx:],
|
||||
]
|
||||
|
||||
|
||||
def bazel_args_with_remote_config(
|
||||
args: Sequence[str], env: Mapping[str, str]
|
||||
) -> list[str]:
|
||||
config = remote_config(args, env)
|
||||
if config is None:
|
||||
return bazel_args_without_remote_execution(args)
|
||||
|
||||
# `remote_config()` returns a configuration only when this key is present.
|
||||
api_key = env["BUILDBUDDY_API_KEY"]
|
||||
remote_args = [
|
||||
f"--config={config}",
|
||||
f"--remote_header=x-buildbuddy-api-key={api_key}",
|
||||
]
|
||||
|
||||
# Insert immediately after the Bazel command. This keeps wrapper-added
|
||||
# options out of positional payloads and lets later CI configs override
|
||||
# shared RBE defaults such as the Windows cross-compilation exec platforms.
|
||||
insertion_idx = next(
|
||||
(idx + 1 for idx, arg in enumerate(args) if not arg.startswith("-")),
|
||||
len(args),
|
||||
)
|
||||
return [*args[:insertion_idx], *remote_args, *args[insertion_idx:]]
|
||||
|
||||
|
||||
def bazel_command(*args: str, env: Mapping[str, str] | None = None) -> list[str]:
|
||||
env = os.environ if env is None else env
|
||||
bazel = env.get("CODEX_BAZEL_BIN", "bazel")
|
||||
return [bazel, *bazel_args_with_remote_config(args, env)]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
config = remote_config(sys.argv[1:], os.environ)
|
||||
if config is None:
|
||||
print(
|
||||
"BuildBuddy key unavailable; using local Bazel configuration.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
else:
|
||||
host_description = (
|
||||
"OpenAI tenant" if uses_openai_host(os.environ) else "generic"
|
||||
)
|
||||
print(
|
||||
f"Using {host_description} BuildBuddy configuration: {config}.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
command = bazel_command(*sys.argv[1:])
|
||||
# Replace the wrapper so Bazel receives signals directly and supplies the
|
||||
# command exit status; a subprocess parent would have no remaining work.
|
||||
os.execvp(command[0], command)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -5,7 +5,6 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -13,6 +12,7 @@ import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
from run_bazel_with_buildbuddy import bazel_command
|
||||
from rusty_v8_module_bazel import (
|
||||
RustyV8ChecksumError,
|
||||
check_module_bazel,
|
||||
@@ -29,33 +29,22 @@ SANDBOX_ARTIFACT_PROFILE = "ptrcomp_sandbox_release"
|
||||
ARTIFACT_BAZEL_CONFIGS = ["rusty-v8-upstream-libcxx"]
|
||||
|
||||
|
||||
def bazel_remote_args() -> list[str]:
|
||||
buildbuddy_api_key = os.environ.get("BUILDBUDDY_API_KEY")
|
||||
if not buildbuddy_api_key:
|
||||
return []
|
||||
return [f"--remote_header=x-buildbuddy-api-key={buildbuddy_api_key}"]
|
||||
|
||||
|
||||
def bazel_execroot() -> Path:
|
||||
result = subprocess.run(
|
||||
["bazel", "info", "execution_root"],
|
||||
output = subprocess.check_output(
|
||||
bazel_command("info", "execution_root"),
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return Path(result.stdout.strip())
|
||||
return Path(output.strip())
|
||||
|
||||
|
||||
def bazel_output_base() -> Path:
|
||||
result = subprocess.run(
|
||||
["bazel", "info", "output_base"],
|
||||
output = subprocess.check_output(
|
||||
bazel_command("info", "output_base"),
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return Path(result.stdout.strip())
|
||||
return Path(output.strip())
|
||||
|
||||
|
||||
def bazel_output_path(path: str) -> Path:
|
||||
@@ -72,24 +61,22 @@ def bazel_output_files(
|
||||
) -> list[Path]:
|
||||
expression = "set(" + " ".join(labels) + ")"
|
||||
bazel_configs = bazel_configs or []
|
||||
result = subprocess.run(
|
||||
[
|
||||
"bazel",
|
||||
output = subprocess.check_output(
|
||||
bazel_command(
|
||||
"cquery",
|
||||
"-c",
|
||||
compilation_mode,
|
||||
f"--platforms=@llvm//platforms:{platform}",
|
||||
*[f"--config={config}" for config in bazel_configs],
|
||||
*bazel_remote_args(),
|
||||
"--output=files",
|
||||
expression,
|
||||
],
|
||||
),
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return [bazel_output_path(line.strip()) for line in result.stdout.splitlines() if line.strip()]
|
||||
return [
|
||||
bazel_output_path(line.strip()) for line in output.splitlines() if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def bazel_build(
|
||||
@@ -102,17 +89,15 @@ def bazel_build(
|
||||
bazel_configs = bazel_configs or []
|
||||
download_args = ["--remote_download_toplevel"] if download_toplevel else []
|
||||
subprocess.run(
|
||||
[
|
||||
"bazel",
|
||||
bazel_command(
|
||||
"build",
|
||||
"-c",
|
||||
compilation_mode,
|
||||
f"--platforms=@llvm//platforms:{platform}",
|
||||
*[f"--config={config}" for config in bazel_configs],
|
||||
*bazel_remote_args(),
|
||||
*download_args,
|
||||
*labels,
|
||||
],
|
||||
),
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
)
|
||||
@@ -172,7 +157,7 @@ def resolved_v8_crate_version() -> str:
|
||||
matches = sorted(
|
||||
set(
|
||||
re.findall(
|
||||
r'https://static\.crates\.io/crates/v8/v8-([0-9]+\.[0-9]+\.[0-9]+)\.crate',
|
||||
r"https://static\.crates\.io/crates/v8/v8-([0-9]+\.[0-9]+\.[0-9]+)\.crate",
|
||||
module_bazel,
|
||||
)
|
||||
)
|
||||
@@ -234,13 +219,17 @@ def stage_artifacts(
|
||||
output_dir: Path,
|
||||
sandbox: bool,
|
||||
) -> None:
|
||||
missing_paths = [str(path) for path in [lib_path, binding_path] if not path.exists()]
|
||||
missing_paths = [
|
||||
str(path) for path in [lib_path, binding_path] if not path.exists()
|
||||
]
|
||||
if missing_paths:
|
||||
raise SystemExit(f"missing release outputs for {target}: {missing_paths}")
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
artifact_profile = SANDBOX_ARTIFACT_PROFILE if sandbox else RELEASE_ARTIFACT_PROFILE
|
||||
staged_library = output_dir / staged_archive_name(target, lib_path, artifact_profile)
|
||||
staged_library = output_dir / staged_archive_name(
|
||||
target, lib_path, artifact_profile
|
||||
)
|
||||
staged_binding = output_dir / staged_binding_name(target, artifact_profile)
|
||||
|
||||
with lib_path.open("rb") as src, staged_library.open("wb") as dst:
|
||||
@@ -270,7 +259,9 @@ def stage_artifacts(
|
||||
|
||||
|
||||
def upstream_release_pair_paths(source_root: Path, target: str) -> tuple[Path, Path]:
|
||||
lib_name = "rusty_v8.lib" if target.endswith("-pc-windows-msvc") else "librusty_v8.a"
|
||||
lib_name = (
|
||||
"rusty_v8.lib" if target.endswith("-pc-windows-msvc") else "librusty_v8.a"
|
||||
)
|
||||
gn_out = source_root / "target" / target / "release" / "gn_out"
|
||||
return gn_out / "obj" / lib_name, gn_out / "src_binding.rs"
|
||||
|
||||
@@ -338,7 +329,9 @@ def parse_args() -> argparse.Namespace:
|
||||
stage_upstream_release_pair_parser = subparsers.add_parser(
|
||||
"stage-upstream-release-pair"
|
||||
)
|
||||
stage_upstream_release_pair_parser.add_argument("--source-root", type=Path, required=True)
|
||||
stage_upstream_release_pair_parser.add_argument(
|
||||
"--source-root", type=Path, required=True
|
||||
)
|
||||
stage_upstream_release_pair_parser.add_argument("--target", required=True)
|
||||
stage_upstream_release_pair_parser.add_argument("--output-dir", required=True)
|
||||
stage_upstream_release_pair_parser.add_argument("--sandbox", action="store_true")
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import run_bazel_with_buildbuddy
|
||||
|
||||
|
||||
class RunBazelWithBuildBuddyTest(unittest.TestCase):
|
||||
def github_env(
|
||||
self,
|
||||
temp_dir: str,
|
||||
*,
|
||||
repository: str = "openai/codex",
|
||||
fork: bool = False,
|
||||
event_name: str = "pull_request",
|
||||
) -> dict[str, str]:
|
||||
event_path = Path(temp_dir) / "event.json"
|
||||
event_path.write_text(
|
||||
json.dumps({"pull_request": {"head": {"repo": {"fork": fork}}}}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return {
|
||||
"BUILDBUDDY_API_KEY": "token",
|
||||
"GITHUB_ACTIONS": "true",
|
||||
"GITHUB_EVENT_NAME": event_name,
|
||||
"GITHUB_EVENT_PATH": str(event_path),
|
||||
"GITHUB_REPOSITORY": repository,
|
||||
}
|
||||
|
||||
def test_keyless_invocation_drops_remote_ci_configuration(self) -> None:
|
||||
self.assertIsNone(
|
||||
run_bazel_with_buildbuddy.remote_config(
|
||||
["build", "--config=ci-linux", "//codex-rs/cli:codex"],
|
||||
{},
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
run_bazel_with_buildbuddy.bazel_args_with_remote_config(
|
||||
["build", "--config=ci-linux", "--", "//codex-rs/cli:codex"],
|
||||
{},
|
||||
),
|
||||
["build", "--", "//codex-rs/cli:codex"],
|
||||
)
|
||||
|
||||
def test_program_arguments_after_separator_do_not_select_or_lose_rbe(self) -> None:
|
||||
args = ["run", "//codex-rs/cli:codex", "--", "--config=remote"]
|
||||
|
||||
self.assertEqual(
|
||||
run_bazel_with_buildbuddy.bazel_args_with_remote_config(args, {}),
|
||||
args,
|
||||
)
|
||||
self.assertEqual(
|
||||
run_bazel_with_buildbuddy.remote_config(
|
||||
args, {"BUILDBUDDY_API_KEY": "fork-token"}
|
||||
),
|
||||
"buildbuddy-generic",
|
||||
)
|
||||
|
||||
def test_upstream_push_selects_openai_rbe_before_target_separator(self) -> None:
|
||||
with TemporaryDirectory() as temp_dir:
|
||||
env = self.github_env(temp_dir, event_name="push")
|
||||
|
||||
self.assertEqual(
|
||||
run_bazel_with_buildbuddy.bazel_args_with_remote_config(
|
||||
["build", "--config=ci-linux", "--", "//codex-rs/cli:codex"],
|
||||
env,
|
||||
),
|
||||
[
|
||||
"build",
|
||||
"--config=buildbuddy-openai-rbe",
|
||||
"--remote_header=x-buildbuddy-api-key=token",
|
||||
"--config=ci-linux",
|
||||
"--",
|
||||
"//codex-rs/cli:codex",
|
||||
],
|
||||
)
|
||||
|
||||
def test_windows_cross_ci_configuration_follows_remote_configuration(self) -> None:
|
||||
env = {"BUILDBUDDY_API_KEY": "fork-token"}
|
||||
|
||||
self.assertEqual(
|
||||
run_bazel_with_buildbuddy.bazel_args_with_remote_config(
|
||||
["build", "--config=ci-windows-cross", "//codex-rs/cli:codex"],
|
||||
env,
|
||||
),
|
||||
[
|
||||
"build",
|
||||
"--config=buildbuddy-generic-rbe",
|
||||
"--remote_header=x-buildbuddy-api-key=fork-token",
|
||||
"--config=ci-windows-cross",
|
||||
"//codex-rs/cli:codex",
|
||||
],
|
||||
)
|
||||
|
||||
def test_query_remote_configuration_is_inserted_before_expression(self) -> None:
|
||||
expression = 'kind("rust_library rule", //codex-rs/...)'
|
||||
env = {"BUILDBUDDY_API_KEY": "fork-token"}
|
||||
|
||||
for command in ("query", "cquery", "aquery"):
|
||||
with self.subTest(command=command):
|
||||
self.assertEqual(
|
||||
run_bazel_with_buildbuddy.bazel_args_with_remote_config(
|
||||
[
|
||||
command,
|
||||
"--config=ci-windows-cross",
|
||||
"--output=label",
|
||||
expression,
|
||||
],
|
||||
env,
|
||||
),
|
||||
[
|
||||
command,
|
||||
"--config=buildbuddy-generic-rbe",
|
||||
"--remote_header=x-buildbuddy-api-key=fork-token",
|
||||
"--config=ci-windows-cross",
|
||||
"--output=label",
|
||||
expression,
|
||||
],
|
||||
)
|
||||
|
||||
def test_same_repository_pull_request_selects_openai_host(self) -> None:
|
||||
with TemporaryDirectory() as temp_dir:
|
||||
self.assertEqual(
|
||||
run_bazel_with_buildbuddy.remote_config(
|
||||
["build", "--config=ci-v8"], self.github_env(temp_dir)
|
||||
),
|
||||
"buildbuddy-openai-rbe",
|
||||
)
|
||||
|
||||
def test_fork_pull_request_cannot_select_openai_host(self) -> None:
|
||||
with TemporaryDirectory() as temp_dir:
|
||||
env = self.github_env(temp_dir, fork=True)
|
||||
|
||||
self.assertEqual(
|
||||
run_bazel_with_buildbuddy.remote_config(
|
||||
["build", "--config=ci-v8"], env
|
||||
),
|
||||
"buildbuddy-generic-rbe",
|
||||
)
|
||||
|
||||
def test_run_in_fork_repository_cannot_select_openai_host(self) -> None:
|
||||
with TemporaryDirectory() as temp_dir:
|
||||
env = self.github_env(temp_dir, repository="contributor/codex")
|
||||
|
||||
self.assertEqual(
|
||||
run_bazel_with_buildbuddy.remote_config(
|
||||
["build", "--config=ci-v8"], env
|
||||
),
|
||||
"buildbuddy-generic-rbe",
|
||||
)
|
||||
|
||||
def test_pull_request_without_readable_event_payload_fails_closed(self) -> None:
|
||||
for event_path in (None, "missing-event.json"):
|
||||
env = {
|
||||
"BUILDBUDDY_API_KEY": "token",
|
||||
"GITHUB_ACTIONS": "true",
|
||||
"GITHUB_EVENT_NAME": "pull_request",
|
||||
"GITHUB_REPOSITORY": "openai/codex",
|
||||
}
|
||||
if event_path is not None:
|
||||
env["GITHUB_EVENT_PATH"] = event_path
|
||||
|
||||
with self.subTest(event_path=event_path):
|
||||
self.assertEqual(
|
||||
run_bazel_with_buildbuddy.remote_config(["build"], env),
|
||||
"buildbuddy-generic",
|
||||
)
|
||||
|
||||
def test_bazel_command_uses_configured_binary_locally(self) -> None:
|
||||
self.assertEqual(
|
||||
run_bazel_with_buildbuddy.bazel_command(
|
||||
"info",
|
||||
"execution_root",
|
||||
env={"CODEX_BAZEL_BIN": "fake-bazel"},
|
||||
),
|
||||
["fake-bazel", "info", "execution_root"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -88,24 +88,49 @@ class RustyV8BazelTest(unittest.TestCase):
|
||||
),
|
||||
)
|
||||
|
||||
def test_bazel_remote_args_include_buildbuddy_header_when_present(self) -> None:
|
||||
with patch.dict(environ, {"BUILDBUDDY_API_KEY": "token"}, clear=False):
|
||||
def test_bazel_commands_use_shared_buildbuddy_remote_config_library(self) -> None:
|
||||
with patch.dict(environ, {}, clear=True):
|
||||
self.assertEqual(
|
||||
["--remote_header=x-buildbuddy-api-key=token"],
|
||||
rusty_v8_bazel.bazel_remote_args(),
|
||||
[
|
||||
"bazel",
|
||||
"build",
|
||||
"//third_party/v8:release",
|
||||
],
|
||||
rusty_v8_bazel.bazel_command(
|
||||
"build",
|
||||
"--config=ci-v8",
|
||||
"//third_party/v8:release",
|
||||
),
|
||||
)
|
||||
with patch.dict(environ, {"BUILDBUDDY_API_KEY": "token"}, clear=True):
|
||||
self.assertEqual(
|
||||
[
|
||||
"bazel",
|
||||
"build",
|
||||
"--config=buildbuddy-generic-rbe",
|
||||
"--remote_header=x-buildbuddy-api-key=token",
|
||||
"--config=ci-v8",
|
||||
"//third_party/v8:release",
|
||||
],
|
||||
rusty_v8_bazel.bazel_command(
|
||||
"build",
|
||||
"--config=ci-v8",
|
||||
"//third_party/v8:release",
|
||||
),
|
||||
)
|
||||
|
||||
with patch.dict(environ, {}, clear=True):
|
||||
self.assertEqual([], rusty_v8_bazel.bazel_remote_args())
|
||||
|
||||
def test_release_pair_labels_and_staged_names_distinguish_sandbox_artifacts(self) -> None:
|
||||
def test_release_pair_labels_and_staged_names_distinguish_sandbox_artifacts(
|
||||
self,
|
||||
) -> None:
|
||||
self.assertEqual(
|
||||
"//third_party/v8:rusty_v8_release_pair_x86_64_unknown_linux_musl",
|
||||
rusty_v8_bazel.release_pair_label("x86_64-unknown-linux-musl"),
|
||||
)
|
||||
self.assertEqual(
|
||||
"//third_party/v8:rusty_v8_sandbox_release_pair_x86_64_unknown_linux_musl",
|
||||
rusty_v8_bazel.release_pair_label("x86_64-unknown-linux-musl", sandbox=True),
|
||||
rusty_v8_bazel.release_pair_label(
|
||||
"x86_64-unknown-linux-musl", sandbox=True
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
"//third_party/v8:rusty_v8_sandbox_release_pair_x86_64_apple_darwin",
|
||||
@@ -205,11 +230,7 @@ class RustyV8BazelTest(unittest.TestCase):
|
||||
with TemporaryDirectory() as source_dir, TemporaryDirectory() as output_dir:
|
||||
source_root = Path(source_dir)
|
||||
gn_out = (
|
||||
source_root
|
||||
/ "target"
|
||||
/ "x86_64-pc-windows-msvc"
|
||||
/ "release"
|
||||
/ "gn_out"
|
||||
source_root / "target" / "x86_64-pc-windows-msvc" / "release" / "gn_out"
|
||||
)
|
||||
(gn_out / "obj").mkdir(parents=True)
|
||||
(gn_out / "obj" / "rusty_v8.lib").write_bytes(b"archive")
|
||||
|
||||
Reference in New Issue
Block a user