Pull plugin service less frequently (#26431)

# Summary
Reduce download traffic to `github.com/openai/plugins` while continuing
to check for updates on every Codex startup.

# Root cause
The startup sync replaced the local repository with a fresh shallow
clone whenever the remote revision changed. At Codex's global scale,
repeatedly downloading the repository created excessive GitHub traffic.

# Changes
- Run `git ls-remote` on each startup to read the remote HEAD SHA.
- Skip all repository downloads when the local and remote SHAs match.
- Update existing checkouts with an exact-SHA shallow `git fetch`,
followed by reset and clean.
- Bootstrap new installations with `git init` plus the same shallow
fetch, rather than cloning.
- Keep the existing file lock so concurrent Codex processes serialize
updates and do not duplicate fetches.
- Preserve the existing GitHub HTTP and export archive fallback
behavior.

# Impact
Each startup makes one lightweight remote HEAD check. Repository objects
are downloaded only when the revision changes, and existing Git objects
are reused during updates.

# Validation
- `just test -p codex-core-plugins startup_sync` (15 tests passed)
- `just test -p codex-core-plugins` (201 tests passed)
- `just clippy -p codex-core-plugins` (passes with one pre-existing
`large_enum_variant` warning)
- Production app-server smoke test against GitHub:
  - Fresh home: `ls-remote`, `git init`, one exact-SHA shallow fetch
- Unchanged restart: `ls-remote` and local `rev-parse` only; no fetch or
clone
- Bench smoke passed
This commit is contained in:
beggers-openai
2026-06-04 17:47:58 -07:00
committed by GitHub
parent 0b2e7b5eb1
commit 72d0bfb6ba
2 changed files with 418 additions and 72 deletions
+127 -14
View File
@@ -1,3 +1,4 @@
use std::fs::File;
use std::path::Path;
use std::path::PathBuf;
use std::process::Command;
@@ -22,8 +23,11 @@ const CURATED_PLUGINS_BACKUP_ARCHIVE_API_URL: &str =
"https://chatgpt.com/backend-api/plugins/export/curated";
const OPENAI_PLUGINS_OWNER: &str = "openai";
const OPENAI_PLUGINS_REPO: &str = "plugins";
const OPENAI_PLUGINS_GIT_URL: &str = "https://github.com/openai/plugins.git";
const CURATED_PLUGINS_FETCH_REF: &str = "refs/codex/curated-sync";
const CURATED_PLUGINS_RELATIVE_DIR: &str = ".tmp/plugins";
const CURATED_PLUGINS_SHA_FILE: &str = ".tmp/plugins.sha";
const CURATED_PLUGINS_SYNC_LOCK_FILE: &str = ".tmp/plugins.sync.lock";
const CURATED_PLUGINS_BACKUP_ARCHIVE_FALLBACK_VERSION: &str = "export-backup";
const CURATED_PLUGINS_GIT_TIMEOUT: Duration = Duration::from_secs(30);
const CURATED_PLUGINS_HTTP_TIMEOUT: Duration = Duration::from_secs(30);
@@ -78,6 +82,8 @@ fn sync_openai_plugins_repo_with_transport_overrides(
api_base_url: &str,
backup_archive_api_url: &str,
) -> Result<String, String> {
let _file_guard = lock_curated_plugins_startup_sync(codex_home)?;
match sync_openai_plugins_repo_via_git(codex_home, git_binary) {
Ok(remote_sha) => {
emit_curated_plugins_startup_sync_metric("git", "success");
@@ -135,6 +141,22 @@ fn sync_openai_plugins_repo_with_transport_overrides(
}
}
fn lock_curated_plugins_startup_sync(codex_home: &Path) -> Result<File, String> {
let lock_path = codex_home.join(CURATED_PLUGINS_SYNC_LOCK_FILE);
std::fs::create_dir_all(codex_home.join(".tmp"))
.map_err(|err| format!("failed to create curated plugins sync directory: {err}"))?;
let lock_file = File::options()
.write(true)
.create(true)
.truncate(false)
.open(&lock_path)
.map_err(|err| format!("failed to open curated plugins sync lock: {err}"))?;
lock_file
.lock()
.map_err(|err| format!("failed to lock curated plugins sync: {err}"))?;
Ok(lock_file)
}
fn sync_openai_plugins_repo_via_git(codex_home: &Path, git_binary: &str) -> Result<String, String> {
let repo_path = curated_plugins_repo_path(codex_home);
let sha_path = codex_home.join(CURATED_PLUGINS_SHA_FILE);
@@ -146,23 +168,30 @@ fn sync_openai_plugins_repo_via_git(codex_home: &Path, git_binary: &str) -> Resu
}
let staged_repo_dir = prepare_curated_repo_parent_and_temp_dir(&repo_path)?;
let clone_output = run_git_command_with_timeout(
Command::new(git_binary)
.env("GIT_OPTIONAL_LOCKS", "0")
.arg("clone")
.arg("--depth")
.arg("1")
.arg("https://github.com/openai/plugins.git")
.arg(staged_repo_dir.path()),
"git clone curated plugins repo",
CURATED_PLUGINS_GIT_TIMEOUT,
run_git_in_repo(
staged_repo_dir.path(),
git_binary,
&["init"],
"git init curated plugins repo",
)?;
ensure_git_success(&clone_output, "git clone curated plugins repo")?;
let cloned_sha = git_head_sha(staged_repo_dir.path(), git_binary)?;
if cloned_sha != remote_sha {
if repo_path.join(".git").is_dir() {
fetch_curated_plugins_commit(&repo_path, &remote_sha, git_binary)?;
fetch_curated_plugins_commit_from_source(
staged_repo_dir.path(),
&repo_path,
CURATED_PLUGINS_FETCH_REF,
git_binary,
)?;
} else {
fetch_curated_plugins_commit(staged_repo_dir.path(), &remote_sha, git_binary)?;
}
reset_curated_plugins_checkout(staged_repo_dir.path(), git_binary)?;
let fetched_sha = git_head_sha(staged_repo_dir.path(), git_binary)?;
if fetched_sha != remote_sha {
return Err(format!(
"curated plugins clone HEAD mismatch: expected {remote_sha}, got {cloned_sha}"
"curated plugins fetch HEAD mismatch: expected {remote_sha}, got {fetched_sha}"
));
}
@@ -172,6 +201,90 @@ fn sync_openai_plugins_repo_via_git(codex_home: &Path, git_binary: &str) -> Resu
Ok(remote_sha)
}
fn fetch_curated_plugins_commit(
repo_path: &Path,
remote_sha: &str,
git_binary: &str,
) -> Result<(), String> {
fetch_curated_plugins_commit_from(
repo_path,
OPENAI_PLUGINS_GIT_URL.as_ref(),
remote_sha,
git_binary,
"git fetch curated plugins repo",
)
}
fn fetch_curated_plugins_commit_from_source(
repo_path: &Path,
source_repo_path: &Path,
remote_sha: &str,
git_binary: &str,
) -> Result<(), String> {
fetch_curated_plugins_commit_from(
repo_path,
source_repo_path,
remote_sha,
git_binary,
"git copy fetched curated plugins commit",
)
}
fn fetch_curated_plugins_commit_from(
repo_path: &Path,
source: &Path,
source_revision: &str,
git_binary: &str,
context: &str,
) -> Result<(), String> {
let fetch_refspec = format!("+{source_revision}:{CURATED_PLUGINS_FETCH_REF}");
let output = run_git_command_with_timeout(
Command::new(git_binary)
.env("GIT_OPTIONAL_LOCKS", "0")
.arg("-C")
.arg(repo_path)
.args(["fetch", "--depth", "1", "--no-tags"])
.arg(source)
.arg(fetch_refspec),
context,
CURATED_PLUGINS_GIT_TIMEOUT,
)?;
ensure_git_success(&output, context)
}
fn reset_curated_plugins_checkout(repo_path: &Path, git_binary: &str) -> Result<(), String> {
run_git_in_repo(
repo_path,
git_binary,
&["reset", "--hard", CURATED_PLUGINS_FETCH_REF],
"git reset curated plugins repo",
)?;
run_git_in_repo(
repo_path,
git_binary,
&["clean", "-fdx"],
"git clean curated plugins repo",
)
}
fn run_git_in_repo(
repo_path: &Path,
git_binary: &str,
args: &[&str],
context: &str,
) -> Result<(), String> {
let output = run_git_command_with_timeout(
Command::new(git_binary)
.env("GIT_OPTIONAL_LOCKS", "0")
.arg("-C")
.arg(repo_path)
.args(args),
context,
CURATED_PLUGINS_GIT_TIMEOUT,
)?;
ensure_git_success(&output, context)
}
fn sync_openai_plugins_repo_via_http(
codex_home: &Path,
api_base_url: &str,