mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
Pull plugin service less frequently (#26431)
# Summary Reduce download traffic to `github.com/openai/plugins` while continuing to check for updates on every Codex startup. # Root cause The startup sync replaced the local repository with a fresh shallow clone whenever the remote revision changed. At Codex's global scale, repeatedly downloading the repository created excessive GitHub traffic. # Changes - Run `git ls-remote` on each startup to read the remote HEAD SHA. - Skip all repository downloads when the local and remote SHAs match. - Update existing checkouts with an exact-SHA shallow `git fetch`, followed by reset and clean. - Bootstrap new installations with `git init` plus the same shallow fetch, rather than cloning. - Keep the existing file lock so concurrent Codex processes serialize updates and do not duplicate fetches. - Preserve the existing GitHub HTTP and export archive fallback behavior. # Impact Each startup makes one lightweight remote HEAD check. Repository objects are downloaded only when the revision changes, and existing Git objects are reused during updates. # Validation - `just test -p codex-core-plugins startup_sync` (15 tests passed) - `just test -p codex-core-plugins` (201 tests passed) - `just clippy -p codex-core-plugins` (passes with one pre-existing `large_enum_variant` warning) - Production app-server smoke test against GitHub: - Fresh home: `ls-remote`, `git init`, one exact-SHA shallow fetch - Unchanged restart: `ls-remote` and local `rev-parse` only; no fetch or clone - Bench smoke passed
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
@@ -22,8 +23,11 @@ const CURATED_PLUGINS_BACKUP_ARCHIVE_API_URL: &str =
|
||||
"https://chatgpt.com/backend-api/plugins/export/curated";
|
||||
const OPENAI_PLUGINS_OWNER: &str = "openai";
|
||||
const OPENAI_PLUGINS_REPO: &str = "plugins";
|
||||
const OPENAI_PLUGINS_GIT_URL: &str = "https://github.com/openai/plugins.git";
|
||||
const CURATED_PLUGINS_FETCH_REF: &str = "refs/codex/curated-sync";
|
||||
const CURATED_PLUGINS_RELATIVE_DIR: &str = ".tmp/plugins";
|
||||
const CURATED_PLUGINS_SHA_FILE: &str = ".tmp/plugins.sha";
|
||||
const CURATED_PLUGINS_SYNC_LOCK_FILE: &str = ".tmp/plugins.sync.lock";
|
||||
const CURATED_PLUGINS_BACKUP_ARCHIVE_FALLBACK_VERSION: &str = "export-backup";
|
||||
const CURATED_PLUGINS_GIT_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const CURATED_PLUGINS_HTTP_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
@@ -78,6 +82,8 @@ fn sync_openai_plugins_repo_with_transport_overrides(
|
||||
api_base_url: &str,
|
||||
backup_archive_api_url: &str,
|
||||
) -> Result<String, String> {
|
||||
let _file_guard = lock_curated_plugins_startup_sync(codex_home)?;
|
||||
|
||||
match sync_openai_plugins_repo_via_git(codex_home, git_binary) {
|
||||
Ok(remote_sha) => {
|
||||
emit_curated_plugins_startup_sync_metric("git", "success");
|
||||
@@ -135,6 +141,22 @@ fn sync_openai_plugins_repo_with_transport_overrides(
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_curated_plugins_startup_sync(codex_home: &Path) -> Result<File, String> {
|
||||
let lock_path = codex_home.join(CURATED_PLUGINS_SYNC_LOCK_FILE);
|
||||
std::fs::create_dir_all(codex_home.join(".tmp"))
|
||||
.map_err(|err| format!("failed to create curated plugins sync directory: {err}"))?;
|
||||
let lock_file = File::options()
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(&lock_path)
|
||||
.map_err(|err| format!("failed to open curated plugins sync lock: {err}"))?;
|
||||
lock_file
|
||||
.lock()
|
||||
.map_err(|err| format!("failed to lock curated plugins sync: {err}"))?;
|
||||
Ok(lock_file)
|
||||
}
|
||||
|
||||
fn sync_openai_plugins_repo_via_git(codex_home: &Path, git_binary: &str) -> Result<String, String> {
|
||||
let repo_path = curated_plugins_repo_path(codex_home);
|
||||
let sha_path = codex_home.join(CURATED_PLUGINS_SHA_FILE);
|
||||
@@ -146,23 +168,30 @@ fn sync_openai_plugins_repo_via_git(codex_home: &Path, git_binary: &str) -> Resu
|
||||
}
|
||||
|
||||
let staged_repo_dir = prepare_curated_repo_parent_and_temp_dir(&repo_path)?;
|
||||
let clone_output = run_git_command_with_timeout(
|
||||
Command::new(git_binary)
|
||||
.env("GIT_OPTIONAL_LOCKS", "0")
|
||||
.arg("clone")
|
||||
.arg("--depth")
|
||||
.arg("1")
|
||||
.arg("https://github.com/openai/plugins.git")
|
||||
.arg(staged_repo_dir.path()),
|
||||
"git clone curated plugins repo",
|
||||
CURATED_PLUGINS_GIT_TIMEOUT,
|
||||
run_git_in_repo(
|
||||
staged_repo_dir.path(),
|
||||
git_binary,
|
||||
&["init"],
|
||||
"git init curated plugins repo",
|
||||
)?;
|
||||
ensure_git_success(&clone_output, "git clone curated plugins repo")?;
|
||||
|
||||
let cloned_sha = git_head_sha(staged_repo_dir.path(), git_binary)?;
|
||||
if cloned_sha != remote_sha {
|
||||
if repo_path.join(".git").is_dir() {
|
||||
fetch_curated_plugins_commit(&repo_path, &remote_sha, git_binary)?;
|
||||
fetch_curated_plugins_commit_from_source(
|
||||
staged_repo_dir.path(),
|
||||
&repo_path,
|
||||
CURATED_PLUGINS_FETCH_REF,
|
||||
git_binary,
|
||||
)?;
|
||||
} else {
|
||||
fetch_curated_plugins_commit(staged_repo_dir.path(), &remote_sha, git_binary)?;
|
||||
}
|
||||
|
||||
reset_curated_plugins_checkout(staged_repo_dir.path(), git_binary)?;
|
||||
let fetched_sha = git_head_sha(staged_repo_dir.path(), git_binary)?;
|
||||
if fetched_sha != remote_sha {
|
||||
return Err(format!(
|
||||
"curated plugins clone HEAD mismatch: expected {remote_sha}, got {cloned_sha}"
|
||||
"curated plugins fetch HEAD mismatch: expected {remote_sha}, got {fetched_sha}"
|
||||
));
|
||||
}
|
||||
|
||||
@@ -172,6 +201,90 @@ fn sync_openai_plugins_repo_via_git(codex_home: &Path, git_binary: &str) -> Resu
|
||||
Ok(remote_sha)
|
||||
}
|
||||
|
||||
fn fetch_curated_plugins_commit(
|
||||
repo_path: &Path,
|
||||
remote_sha: &str,
|
||||
git_binary: &str,
|
||||
) -> Result<(), String> {
|
||||
fetch_curated_plugins_commit_from(
|
||||
repo_path,
|
||||
OPENAI_PLUGINS_GIT_URL.as_ref(),
|
||||
remote_sha,
|
||||
git_binary,
|
||||
"git fetch curated plugins repo",
|
||||
)
|
||||
}
|
||||
|
||||
fn fetch_curated_plugins_commit_from_source(
|
||||
repo_path: &Path,
|
||||
source_repo_path: &Path,
|
||||
remote_sha: &str,
|
||||
git_binary: &str,
|
||||
) -> Result<(), String> {
|
||||
fetch_curated_plugins_commit_from(
|
||||
repo_path,
|
||||
source_repo_path,
|
||||
remote_sha,
|
||||
git_binary,
|
||||
"git copy fetched curated plugins commit",
|
||||
)
|
||||
}
|
||||
|
||||
fn fetch_curated_plugins_commit_from(
|
||||
repo_path: &Path,
|
||||
source: &Path,
|
||||
source_revision: &str,
|
||||
git_binary: &str,
|
||||
context: &str,
|
||||
) -> Result<(), String> {
|
||||
let fetch_refspec = format!("+{source_revision}:{CURATED_PLUGINS_FETCH_REF}");
|
||||
let output = run_git_command_with_timeout(
|
||||
Command::new(git_binary)
|
||||
.env("GIT_OPTIONAL_LOCKS", "0")
|
||||
.arg("-C")
|
||||
.arg(repo_path)
|
||||
.args(["fetch", "--depth", "1", "--no-tags"])
|
||||
.arg(source)
|
||||
.arg(fetch_refspec),
|
||||
context,
|
||||
CURATED_PLUGINS_GIT_TIMEOUT,
|
||||
)?;
|
||||
ensure_git_success(&output, context)
|
||||
}
|
||||
|
||||
fn reset_curated_plugins_checkout(repo_path: &Path, git_binary: &str) -> Result<(), String> {
|
||||
run_git_in_repo(
|
||||
repo_path,
|
||||
git_binary,
|
||||
&["reset", "--hard", CURATED_PLUGINS_FETCH_REF],
|
||||
"git reset curated plugins repo",
|
||||
)?;
|
||||
run_git_in_repo(
|
||||
repo_path,
|
||||
git_binary,
|
||||
&["clean", "-fdx"],
|
||||
"git clean curated plugins repo",
|
||||
)
|
||||
}
|
||||
|
||||
fn run_git_in_repo(
|
||||
repo_path: &Path,
|
||||
git_binary: &str,
|
||||
args: &[&str],
|
||||
context: &str,
|
||||
) -> Result<(), String> {
|
||||
let output = run_git_command_with_timeout(
|
||||
Command::new(git_binary)
|
||||
.env("GIT_OPTIONAL_LOCKS", "0")
|
||||
.arg("-C")
|
||||
.arg(repo_path)
|
||||
.args(args),
|
||||
context,
|
||||
CURATED_PLUGINS_GIT_TIMEOUT,
|
||||
)?;
|
||||
ensure_git_success(&output, context)
|
||||
}
|
||||
|
||||
fn sync_openai_plugins_repo_via_http(
|
||||
codex_home: &Path,
|
||||
api_base_url: &str,
|
||||
|
||||
Reference in New Issue
Block a user