From 03edd4fbee45e1f4fa736f1077b2df312d286f60 Mon Sep 17 00:00:00 2001 From: xl-openai Date: Mon, 6 Apr 2026 15:36:20 -0700 Subject: [PATCH] feat: fallback curated plugin download from backend endpint. (#16947) Add one more fallback for downloading the curated plugin repo from chatgpt.com. Have to be the last fallback for now as it is a lagging backup. --- codex-rs/core/src/plugins/manager.rs | 20 +- codex-rs/core/src/plugins/startup_sync.rs | 261 +++++++- .../core/src/plugins/startup_sync_tests.rs | 595 +++++++++++++----- 3 files changed, 691 insertions(+), 185 deletions(-) diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index 2b21a2370..388c677e1 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -719,6 +719,7 @@ impl PluginsManager { )); } + let mut missing_remote_plugins = Vec::::new(); let mut remote_installed_plugin_names = HashSet::::new(); for plugin in remote_plugins { if plugin.marketplace_name != marketplace_name { @@ -727,11 +728,7 @@ impl PluginsManager { }); } if !local_plugin_names.contains(&plugin.name) { - warn!( - plugin = plugin.name, - marketplace = %marketplace_name, - "ignoring remote plugin missing from local marketplace during sync" - ); + missing_remote_plugins.push(plugin.name); continue; } // For now, sync treats remote `enabled = false` as uninstall rather than a distinct @@ -753,6 +750,19 @@ impl PluginsManager { let mut result = RemotePluginSyncResult::default(); let remote_plugin_count = remote_installed_plugin_names.len(); let local_plugin_count = local_plugins.len(); + if !missing_remote_plugins.is_empty() { + let sample_missing_plugins = missing_remote_plugins + .iter() + .take(10) + .cloned() + .collect::>(); + warn!( + marketplace = %marketplace_name, + missing_remote_plugin_count = missing_remote_plugins.len(), + missing_remote_plugin_examples = ?sample_missing_plugins, + "ignoring remote plugins missing from local marketplace during sync" + ); + } for ( plugin_name, diff --git a/codex-rs/core/src/plugins/startup_sync.rs b/codex-rs/core/src/plugins/startup_sync.rs index 6dcf18f54..14608d131 100644 --- a/codex-rs/core/src/plugins/startup_sync.rs +++ b/codex-rs/core/src/plugins/startup_sync.rs @@ -24,16 +24,20 @@ use super::PluginsManager; const GITHUB_API_BASE_URL: &str = "https://api.github.com"; const GITHUB_API_ACCEPT_HEADER: &str = "application/vnd.github+json"; const GITHUB_API_VERSION_HEADER: &str = "2022-11-28"; +const CURATED_PLUGINS_BACKUP_ARCHIVE_API_URL: &str = + "https://chatgpt.com/backend-api/plugins/export/curated"; const OPENAI_PLUGINS_OWNER: &str = "openai"; const OPENAI_PLUGINS_REPO: &str = "plugins"; const CURATED_PLUGINS_RELATIVE_DIR: &str = ".tmp/plugins"; const CURATED_PLUGINS_SHA_FILE: &str = ".tmp/plugins.sha"; +const CURATED_PLUGINS_BACKUP_ARCHIVE_FALLBACK_VERSION: &str = "export-backup"; const CURATED_PLUGINS_GIT_TIMEOUT: Duration = Duration::from_secs(30); const CURATED_PLUGINS_HTTP_TIMEOUT: Duration = Duration::from_secs(30); +const CURATED_PLUGINS_BACKUP_ARCHIVE_TIMEOUT: Duration = Duration::from_secs(30); // Keep this comfortably above a normal sync attempt so we do not race another Codex process. const CURATED_PLUGINS_STALE_TEMP_DIR_MAX_AGE: Duration = Duration::from_secs(10 * 60); const STARTUP_REMOTE_PLUGIN_SYNC_MARKER_FILE: &str = ".tmp/app-server-remote-plugin-sync-v1"; -const STARTUP_REMOTE_PLUGIN_SYNC_PREREQUISITE_TIMEOUT: Duration = Duration::from_secs(5); +const STARTUP_REMOTE_PLUGIN_SYNC_PREREQUISITE_TIMEOUT: Duration = Duration::from_secs(10); #[derive(Debug, Deserialize)] struct GitHubRepositorySummary { @@ -50,22 +54,37 @@ struct GitHubGitRefObject { sha: String, } +#[derive(Debug, Deserialize)] +struct CuratedPluginsBackupArchiveResponse { + download_url: String, +} + pub(crate) fn curated_plugins_repo_path(codex_home: &Path) -> PathBuf { codex_home.join(CURATED_PLUGINS_RELATIVE_DIR) } pub(crate) fn read_curated_plugins_sha(codex_home: &Path) -> Option { - read_sha_file(codex_home.join(CURATED_PLUGINS_SHA_FILE).as_path()) + read_sha_file(curated_plugins_sha_path(codex_home).as_path()) +} + +fn curated_plugins_sha_path(codex_home: &Path) -> PathBuf { + codex_home.join(CURATED_PLUGINS_SHA_FILE) } pub(crate) fn sync_openai_plugins_repo(codex_home: &Path) -> Result { - sync_openai_plugins_repo_with_transport_overrides(codex_home, "git", GITHUB_API_BASE_URL) + sync_openai_plugins_repo_with_transport_overrides( + codex_home, + "git", + GITHUB_API_BASE_URL, + CURATED_PLUGINS_BACKUP_ARCHIVE_API_URL, + ) } fn sync_openai_plugins_repo_with_transport_overrides( codex_home: &Path, git_binary: &str, api_base_url: &str, + backup_archive_api_url: &str, ) -> Result { match sync_openai_plugins_repo_via_git(codex_home, git_binary) { Ok(remote_sha) => { @@ -80,11 +99,46 @@ fn sync_openai_plugins_repo_with_transport_overrides( git_binary, "git sync failed for curated plugin sync; falling back to GitHub HTTP" ); - let result = sync_openai_plugins_repo_via_http(codex_home, api_base_url); - let status = if result.is_ok() { "success" } else { "failure" }; - emit_curated_plugins_startup_sync_metric("http", status); - emit_curated_plugins_startup_sync_final_metric("http", status); - result + match sync_openai_plugins_repo_via_http(codex_home, api_base_url) { + Ok(remote_sha) => { + emit_curated_plugins_startup_sync_metric("http", "success"); + emit_curated_plugins_startup_sync_final_metric("http", "success"); + Ok(remote_sha) + } + Err(http_err) => { + emit_curated_plugins_startup_sync_metric("http", "failure"); + if has_local_curated_plugins_snapshot(codex_home) { + emit_curated_plugins_startup_sync_final_metric("http", "failure"); + warn!( + error = %http_err, + "GitHub HTTP sync failed for curated plugin sync; skipping export archive fallback because a local curated plugins snapshot already exists" + ); + Err(format!( + "git sync failed for curated plugin sync: {err}; GitHub HTTP sync failed for curated plugin sync: {http_err}; export archive fallback skipped because a local curated plugins snapshot already exists" + )) + } else { + // The export archive is a lagging backup path. Only use it to bootstrap a + // missing local curated snapshot, never to refresh an existing one. + warn!( + error = %http_err, + backup_archive_api_url, + "GitHub HTTP sync failed for curated plugin sync; falling back to export archive" + ); + let result = sync_openai_plugins_repo_via_backup_archive( + codex_home, + backup_archive_api_url, + ); + let status = if result.is_ok() { "success" } else { "failure" }; + emit_curated_plugins_startup_sync_metric("export_archive", status); + emit_curated_plugins_startup_sync_final_metric("export_archive", status); + result.map_err(|export_err| { + format!( + "git sync failed for curated plugin sync: {err}; GitHub HTTP sync failed for curated plugin sync: {http_err}; export archive sync failed for curated plugin sync: {export_err}" + ) + }) + } + } + } } } } @@ -152,6 +206,29 @@ fn sync_openai_plugins_repo_via_http( Ok(remote_sha) } +fn sync_openai_plugins_repo_via_backup_archive( + codex_home: &Path, + backup_archive_api_url: &str, +) -> Result { + let repo_path = curated_plugins_repo_path(codex_home); + let sha_path = curated_plugins_sha_path(codex_home); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|err| format!("failed to create curated plugins sync runtime: {err}"))?; + let staged_repo_dir = prepare_curated_repo_parent_and_temp_dir(&repo_path)?; + let zipball_bytes = runtime.block_on(fetch_curated_repo_backup_archive_zip( + backup_archive_api_url, + ))?; + extract_zipball_to_dir(&zipball_bytes, staged_repo_dir.path())?; + ensure_marketplace_manifest_exists(staged_repo_dir.path())?; + let export_version = read_extracted_backup_archive_git_sha(staged_repo_dir.path())? + .unwrap_or_else(|| CURATED_PLUGINS_BACKUP_ARCHIVE_FALLBACK_VERSION.to_string()); + activate_curated_repo(&repo_path, staged_repo_dir)?; + write_curated_plugins_sha(&sha_path, &export_version)?; + Ok(export_version) +} + pub(super) fn start_startup_remote_plugin_sync_once( manager: Arc, codex_home: PathBuf, @@ -213,17 +290,17 @@ fn startup_remote_plugin_sync_marker_path(codex_home: &Path) -> PathBuf { codex_home.join(STARTUP_REMOTE_PLUGIN_SYNC_MARKER_FILE) } -fn startup_remote_plugin_sync_prerequisites_ready(codex_home: &Path) -> bool { - codex_home - .join(".tmp/plugins/.agents/plugins/marketplace.json") +fn has_local_curated_plugins_snapshot(codex_home: &Path) -> bool { + curated_plugins_repo_path(codex_home) + .join(".agents/plugins/marketplace.json") .is_file() - && codex_home.join(".tmp/plugins.sha").is_file() + && codex_home.join(CURATED_PLUGINS_SHA_FILE).is_file() } async fn wait_for_startup_remote_plugin_sync_prerequisites(codex_home: &Path) -> bool { let deadline = tokio::time::Instant::now() + STARTUP_REMOTE_PLUGIN_SYNC_PREREQUISITE_TIMEOUT; loop { - if startup_remote_plugin_sync_prerequisites_ready(codex_home) { + if has_local_curated_plugins_snapshot(codex_home) { return true; } if tokio::time::Instant::now() >= deadline { @@ -641,6 +718,126 @@ async fn fetch_curated_repo_zipball( fetch_github_bytes(&client, &zipball_url, "download curated plugins archive").await } +async fn fetch_curated_repo_backup_archive_zip( + backup_archive_api_url: &str, +) -> Result, String> { + let client = build_reqwest_client(); + let export_body = fetch_public_text( + &client, + backup_archive_api_url, + "get curated plugins export archive metadata", + ) + .await?; + let export_response: CuratedPluginsBackupArchiveResponse = serde_json::from_str(&export_body) + .map_err(|err| { + format!( + "failed to parse curated plugins backup archive response from {backup_archive_api_url}: {err}" + ) + })?; + if export_response.download_url.is_empty() { + return Err(format!( + "curated plugins backup archive response from {backup_archive_api_url} did not include a download URL" + )); + } + + fetch_public_bytes( + &client, + &export_response.download_url, + "download curated plugins export archive", + ) + .await +} + +fn read_extracted_backup_archive_git_sha(repo_path: &Path) -> Result, String> { + let git_dir = repo_path.join(".git"); + if !git_dir.is_dir() { + return Ok(None); + } + + let head_path = git_dir.join("HEAD"); + let head = std::fs::read_to_string(&head_path).map_err(|err| { + format!( + "failed to read curated plugins backup archive git HEAD {}: {err}", + head_path.display() + ) + })?; + let head = head.trim(); + if head.is_empty() { + return Err(format!( + "curated plugins backup archive git HEAD is empty at {}", + head_path.display() + )); + } + + if let Some(reference) = head.strip_prefix("ref: ") { + let reference = validate_backup_archive_git_ref(reference.trim())?; + return read_git_ref_sha(&git_dir, reference).map(Some); + } + + Ok(Some(head.to_string())) +} + +fn validate_backup_archive_git_ref(reference: &str) -> Result<&str, String> { + if !reference.starts_with("refs/") { + return Err(format!( + "curated plugins backup archive git ref must stay under refs/: {reference}" + )); + } + + let path = Path::new(reference); + if path.is_absolute() { + return Err(format!( + "curated plugins backup archive git ref must be relative: {reference}" + )); + } + + for component in path.components() { + match component { + std::path::Component::Normal(_) => {} + _ => { + return Err(format!( + "curated plugins backup archive git ref contains invalid path components: {reference}" + )); + } + } + } + + Ok(reference) +} + +fn read_git_ref_sha(git_dir: &Path, reference: &str) -> Result { + let ref_path = git_dir.join(reference); + if let Ok(sha) = std::fs::read_to_string(&ref_path) { + let sha = sha.trim(); + if sha.is_empty() { + return Err(format!( + "curated plugins backup archive git ref {reference} is empty at {}", + ref_path.display() + )); + } + return Ok(sha.to_string()); + } + + let packed_refs_path = git_dir.join("packed-refs"); + if let Ok(packed_refs) = std::fs::read_to_string(&packed_refs_path) + && let Some(sha) = packed_refs.lines().find_map(|line| { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') || trimmed.starts_with('^') { + return None; + } + let (sha, candidate_ref) = trimmed.split_once(' ')?; + (candidate_ref == reference).then_some(sha.to_string()) + }) + { + return Ok(sha); + } + + Err(format!( + "failed to resolve curated plugins backup archive git ref {reference} from {}", + git_dir.display() + )) +} + async fn fetch_github_text(client: &Client, url: &str, context: &str) -> Result { let response = github_request(client, url) .send() @@ -675,6 +872,44 @@ async fn fetch_github_bytes(client: &Client, url: &str, context: &str) -> Result Ok(body.to_vec()) } +async fn fetch_public_text(client: &Client, url: &str, context: &str) -> Result { + let response = client + .get(url) + .timeout(CURATED_PLUGINS_BACKUP_ARCHIVE_TIMEOUT) + .send() + .await + .map_err(|err| format!("failed to {context} from {url}: {err}"))?; + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + if !status.is_success() { + return Err(format!( + "{context} from {url} failed with status {status}: {body}" + )); + } + Ok(body) +} + +async fn fetch_public_bytes(client: &Client, url: &str, context: &str) -> Result, String> { + let response = client + .get(url) + .timeout(CURATED_PLUGINS_BACKUP_ARCHIVE_TIMEOUT) + .send() + .await + .map_err(|err| format!("failed to {context} from {url}: {err}"))?; + let status = response.status(); + let body = response + .bytes() + .await + .map_err(|err| format!("failed to read {context} response from {url}: {err}"))?; + if !status.is_success() { + let body_text = String::from_utf8_lossy(&body); + return Err(format!( + "{context} from {url} failed with status {status}: {body_text}" + )); + } + Ok(body.to_vec()) +} + fn github_request(client: &Client, url: &str) -> reqwest::RequestBuilder { client .get(url) diff --git a/codex-rs/core/src/plugins/startup_sync_tests.rs b/codex-rs/core/src/plugins/startup_sync_tests.rs index e1a1d8596..cd0b91f4d 100644 --- a/codex-rs/core/src/plugins/startup_sync_tests.rs +++ b/codex-rs/core/src/plugins/startup_sync_tests.rs @@ -8,6 +8,7 @@ use codex_login::CodexAuth; use pretty_assertions::assert_eq; use std::io::Write; use std::path::Path; +use std::path::PathBuf; use tempfile::tempdir; use wiremock::Mock; use wiremock::MockServer; @@ -33,6 +34,111 @@ fn has_plugins_clone_dirs(codex_home: &Path) -> bool { }) } +fn write_executable_script(path: &Path, contents: &str) { + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt; + + std::fs::write(path, contents).expect("write script"); + #[cfg(unix)] + { + let mut permissions = std::fs::metadata(path).expect("metadata").permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(path, permissions).expect("chmod"); + } +} + +async fn mount_github_repo_and_ref(server: &MockServer, sha: &str) { + Mock::given(method("GET")) + .and(path("/repos/openai/plugins")) + .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"default_branch":"main"}"#)) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path("/repos/openai/plugins/git/ref/heads/main")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(format!(r#"{{"object":{{"sha":"{sha}"}}}}"#)), + ) + .mount(server) + .await; +} + +async fn mount_github_zipball(server: &MockServer, sha: &str, bytes: Vec) { + Mock::given(method("GET")) + .and(path(format!("/repos/openai/plugins/zipball/{sha}"))) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/zip") + .set_body_bytes(bytes), + ) + .mount(server) + .await; +} + +async fn mount_export_archive(server: &MockServer, bytes: Vec) -> String { + let export_api_url = format!("{}/backend-api/plugins/export/curated", server.uri()); + Mock::given(method("GET")) + .and(path("/backend-api/plugins/export/curated")) + .respond_with(ResponseTemplate::new(200).set_body_string(format!( + r#"{{"download_url":"{}/files/curated-plugins.zip"}}"#, + server.uri() + ))) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path("/files/curated-plugins.zip")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/zip") + .set_body_bytes(bytes), + ) + .mount(server) + .await; + export_api_url +} + +async fn run_sync_with_transport_overrides( + codex_home: PathBuf, + git_binary: impl Into, + api_base_url: impl Into, + backup_archive_api_url: impl Into, +) -> Result { + let git_binary = git_binary.into(); + let api_base_url = api_base_url.into(); + let backup_archive_api_url = backup_archive_api_url.into(); + tokio::task::spawn_blocking(move || { + sync_openai_plugins_repo_with_transport_overrides( + codex_home.as_path(), + &git_binary, + &api_base_url, + &backup_archive_api_url, + ) + }) + .await + .expect("sync task should join") +} + +async fn run_http_sync( + codex_home: PathBuf, + api_base_url: impl Into, +) -> Result { + let api_base_url = api_base_url.into(); + tokio::task::spawn_blocking(move || { + sync_openai_plugins_repo_via_http(codex_home.as_path(), &api_base_url) + }) + .await + .expect("sync task should join") +} + +fn assert_curated_gmail_repo(repo_path: &Path) { + assert!(repo_path.join(".agents/plugins/marketplace.json").is_file()); + assert!( + repo_path + .join("plugins/gmail/.codex-plugin/plugin.json") + .is_file() + ); +} + #[test] fn curated_plugins_repo_path_uses_codex_home_tmp_dir() { let tmp = tempdir().expect("tempdir"); @@ -100,8 +206,6 @@ fn remove_stale_curated_repo_temp_dirs_removes_only_matching_directories() { #[cfg(unix)] #[test] fn sync_openai_plugins_repo_prefers_git_when_available() { - use std::os::unix::fs::PermissionsExt; - let tmp = tempdir().expect("tempdir"); let bin_dir = tempfile::Builder::new() .prefix("fake-git-") @@ -110,9 +214,9 @@ fn sync_openai_plugins_repo_prefers_git_when_available() { let git_path = bin_dir.path().join("git"); let sha = "0123456789abcdef0123456789abcdef01234567"; - std::fs::write( + write_executable_script( &git_path, - format!( + &format!( r#"#!/bin/sh if [ "$1" = "ls-remote" ]; then printf '%s\tHEAD\n' "{sha}" @@ -135,89 +239,164 @@ echo "unexpected git invocation: $@" >&2 exit 1 "# ), - ) - .expect("write fake git"); - let mut permissions = std::fs::metadata(&git_path) - .expect("metadata") - .permissions(); - permissions.set_mode(0o755); - std::fs::set_permissions(&git_path, permissions).expect("chmod"); + ); let synced_sha = sync_openai_plugins_repo_with_transport_overrides( tmp.path(), git_path.to_str().expect("utf8 path"), "http://127.0.0.1:9", + "http://127.0.0.1:9/backend-api/plugins/export/curated", ) .expect("git sync should succeed"); assert_eq!(synced_sha, sha); - assert!(curated_plugins_repo_path(tmp.path()).join(".git").is_dir()); - assert!( - curated_plugins_repo_path(tmp.path()) - .join(".agents/plugins/marketplace.json") - .is_file() - ); + let repo_path = curated_plugins_repo_path(tmp.path()); + assert!(repo_path.join(".git").is_dir()); + assert_curated_gmail_repo(&repo_path); assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); } +#[cfg(unix)] +#[test] +fn sync_openai_plugins_repo_via_git_succeeds_with_local_rewritten_remote() { + let tmp = tempdir().expect("tempdir"); + let repo_root = tempfile::Builder::new() + .prefix("curated-repo-success-") + .tempdir() + .expect("tempdir"); + let work_repo = repo_root.path().join("work/plugins"); + let remote_repo = repo_root.path().join("remotes/openai/plugins.git"); + std::fs::create_dir_all(work_repo.join(".agents/plugins")).expect("create marketplace dir"); + std::fs::create_dir_all(work_repo.join("plugins/gmail/.codex-plugin")) + .expect("create plugin dir"); + std::fs::write( + work_repo.join(".agents/plugins/marketplace.json"), + r#"{"name":"openai-curated","plugins":[{"name":"gmail","source":{"source":"local","path":"./plugins/gmail"}}]}"#, + ) + .expect("write marketplace"); + std::fs::write( + work_repo.join("plugins/gmail/.codex-plugin/plugin.json"), + r#"{"name":"gmail"}"#, + ) + .expect("write plugin manifest"); + + let init_status = Command::new("git") + .arg("-C") + .arg(&work_repo) + .arg("init") + .status() + .expect("run git init"); + assert!(init_status.success()); + + let add_status = Command::new("git") + .arg("-C") + .arg(&work_repo) + .arg("add") + .arg(".") + .status() + .expect("run git add"); + assert!(add_status.success()); + + let commit_status = Command::new("git") + .arg("-C") + .arg(&work_repo) + .arg("-c") + .arg("user.name=Codex Test") + .arg("-c") + .arg("user.email=codex@example.com") + .arg("commit") + .arg("-m") + .arg("init") + .status() + .expect("run git commit"); + assert!(commit_status.success()); + + std::fs::create_dir_all(remote_repo.parent().expect("remote parent")) + .expect("create remote parent"); + let clone_status = Command::new("git") + .arg("clone") + .arg("--bare") + .arg(&work_repo) + .arg(&remote_repo) + .status() + .expect("run git clone --bare"); + assert!(clone_status.success()); + + let sha_output = Command::new("git") + .arg("-C") + .arg(&work_repo) + .arg("rev-parse") + .arg("HEAD") + .output() + .expect("run git rev-parse"); + assert!(sha_output.status.success()); + let sha = String::from_utf8_lossy(&sha_output.stdout) + .trim() + .to_string(); + + let git_config_path = repo_root.path().join("git-rewrite.conf"); + std::fs::write( + &git_config_path, + format!( + "[url \"file://{}/\"]\n insteadOf = https://github.com/\n", + repo_root.path().join("remotes").display() + ), + ) + .expect("write git config"); + + let bin_dir = tempfile::Builder::new() + .prefix("git-rewrite-wrapper-") + .tempdir() + .expect("tempdir"); + let git_wrapper = bin_dir.path().join("git"); + write_executable_script( + &git_wrapper, + &format!( + "#!/bin/sh\nGIT_CONFIG_GLOBAL='{}' exec git \"$@\"\n", + git_config_path.display() + ), + ); + + let synced_sha = + sync_openai_plugins_repo_via_git(tmp.path(), git_wrapper.to_str().expect("utf8 path")) + .expect("git sync should succeed"); + + assert_eq!(synced_sha, sha); + assert_curated_gmail_repo(&curated_plugins_repo_path(tmp.path())); + assert_eq!( + read_curated_plugins_sha(tmp.path()).as_deref(), + Some(sha.as_str()) + ); + assert!(!has_plugins_clone_dirs(tmp.path())); +} + #[tokio::test] async fn sync_openai_plugins_repo_falls_back_to_http_when_git_is_unavailable() { let tmp = tempdir().expect("tempdir"); let server = MockServer::start().await; let sha = "0123456789abcdef0123456789abcdef01234567"; - Mock::given(method("GET")) - .and(path("/repos/openai/plugins")) - .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"default_branch":"main"}"#)) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/repos/openai/plugins/git/ref/heads/main")) - .respond_with( - ResponseTemplate::new(200) - .set_body_string(format!(r#"{{"object":{{"sha":"{sha}"}}}}"#)), - ) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path(format!("/repos/openai/plugins/zipball/{sha}"))) - .respond_with( - ResponseTemplate::new(200) - .insert_header("content-type", "application/zip") - .set_body_bytes(curated_repo_zipball_bytes(sha)), - ) - .mount(&server) - .await; + mount_github_repo_and_ref(&server, sha).await; + mount_github_zipball(&server, sha, curated_repo_zipball_bytes(sha)).await; - let server_uri = server.uri(); - let tmp_path = tmp.path().to_path_buf(); - let synced_sha = tokio::task::spawn_blocking(move || { - sync_openai_plugins_repo_with_transport_overrides( - tmp_path.as_path(), - "missing-git-for-test", - &server_uri, - ) - }) + let synced_sha = run_sync_with_transport_overrides( + tmp.path().to_path_buf(), + "missing-git-for-test", + server.uri(), + "http://127.0.0.1:9/backend-api/plugins/export/curated", + ) .await - .expect("sync task should join") .expect("fallback sync should succeed"); let repo_path = curated_plugins_repo_path(tmp.path()); assert_eq!(synced_sha, sha); - assert!(repo_path.join(".agents/plugins/marketplace.json").is_file()); - assert!( - repo_path - .join("plugins/gmail/.codex-plugin/plugin.json") - .is_file() - ); + assert_curated_gmail_repo(&repo_path); assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); } #[cfg(unix)] #[tokio::test] async fn sync_openai_plugins_repo_falls_back_to_http_when_git_sync_fails() { - use std::os::unix::fs::PermissionsExt; - let tmp = tempdir().expect("tempdir"); let bin_dir = tempfile::Builder::new() .prefix("fake-git-fail-") @@ -226,73 +405,36 @@ async fn sync_openai_plugins_repo_falls_back_to_http_when_git_sync_fails() { let git_path = bin_dir.path().join("git"); let sha = "0123456789abcdef0123456789abcdef01234567"; - std::fs::write( + write_executable_script( &git_path, r#"#!/bin/sh echo "simulated git failure" >&2 exit 1 "#, - ) - .expect("write fake git"); - let mut permissions = std::fs::metadata(&git_path) - .expect("metadata") - .permissions(); - permissions.set_mode(0o755); - std::fs::set_permissions(&git_path, permissions).expect("chmod"); + ); let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/repos/openai/plugins")) - .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"default_branch":"main"}"#)) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/repos/openai/plugins/git/ref/heads/main")) - .respond_with( - ResponseTemplate::new(200) - .set_body_string(format!(r#"{{"object":{{"sha":"{sha}"}}}}"#)), - ) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path(format!("/repos/openai/plugins/zipball/{sha}"))) - .respond_with( - ResponseTemplate::new(200) - .insert_header("content-type", "application/zip") - .set_body_bytes(curated_repo_zipball_bytes(sha)), - ) - .mount(&server) - .await; + mount_github_repo_and_ref(&server, sha).await; + mount_github_zipball(&server, sha, curated_repo_zipball_bytes(sha)).await; - let server_uri = server.uri(); - let tmp_path = tmp.path().to_path_buf(); - let synced_sha = tokio::task::spawn_blocking(move || { - sync_openai_plugins_repo_with_transport_overrides( - tmp_path.as_path(), - git_path.to_str().expect("utf8 path"), - &server_uri, - ) - }) + let synced_sha = run_sync_with_transport_overrides( + tmp.path().to_path_buf(), + git_path.to_str().expect("utf8 path"), + server.uri(), + "http://127.0.0.1:9/backend-api/plugins/export/curated", + ) .await - .expect("sync task should join") .expect("fallback sync should succeed"); let repo_path = curated_plugins_repo_path(tmp.path()); assert_eq!(synced_sha, sha); - assert!(repo_path.join(".agents/plugins/marketplace.json").is_file()); - assert!( - repo_path - .join("plugins/gmail/.codex-plugin/plugin.json") - .is_file() - ); + assert_curated_gmail_repo(&repo_path); assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); } #[cfg(unix)] #[test] fn sync_openai_plugins_repo_via_git_cleans_up_staged_dir_on_clone_failure() { - use std::os::unix::fs::PermissionsExt; - let tmp = tempdir().expect("tempdir"); let bin_dir = tempfile::Builder::new() .prefix("fake-git-partial-fail-") @@ -301,9 +443,9 @@ fn sync_openai_plugins_repo_via_git_cleans_up_staged_dir_on_clone_failure() { let git_path = bin_dir.path().join("git"); let sha = "0123456789abcdef0123456789abcdef01234567"; - std::fs::write( + write_executable_script( &git_path, - format!( + &format!( r#"#!/bin/sh if [ "$1" = "ls-remote" ]; then printf '%s\tHEAD\n' "{sha}" @@ -319,13 +461,7 @@ echo "unexpected git invocation: $@" >&2 exit 1 "# ), - ) - .expect("write fake git"); - let mut permissions = std::fs::metadata(&git_path) - .expect("metadata") - .permissions(); - permissions.set_mode(0o755); - std::fs::set_permissions(&git_path, permissions).expect("chmod"); + ); let err = sync_openai_plugins_repo_via_git(tmp.path(), git_path.to_str().expect("utf8 path")) .expect_err("git sync should fail"); @@ -340,37 +476,12 @@ async fn sync_openai_plugins_repo_via_http_cleans_up_staged_dir_on_extract_failu let server = MockServer::start().await; let sha = "0123456789abcdef0123456789abcdef01234567"; - Mock::given(method("GET")) - .and(path("/repos/openai/plugins")) - .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"default_branch":"main"}"#)) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/repos/openai/plugins/git/ref/heads/main")) - .respond_with( - ResponseTemplate::new(200) - .set_body_string(format!(r#"{{"object":{{"sha":"{sha}"}}}}"#)), - ) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path(format!("/repos/openai/plugins/zipball/{sha}"))) - .respond_with( - ResponseTemplate::new(200) - .insert_header("content-type", "application/zip") - .set_body_bytes(b"not a zip archive".to_vec()), - ) - .mount(&server) - .await; + mount_github_repo_and_ref(&server, sha).await; + mount_github_zipball(&server, sha, b"not a zip archive".to_vec()).await; - let server_uri = server.uri(); - let tmp_path = tmp.path().to_path_buf(); - let err = tokio::task::spawn_blocking(move || { - sync_openai_plugins_repo_via_http(tmp_path.as_path(), &server_uri) - }) - .await - .expect("sync task should join") - .expect_err("http sync should fail"); + let err = run_http_sync(tmp.path().to_path_buf(), server.uri()) + .await + .expect_err("http sync should fail"); assert!(err.contains("failed to open curated plugins zip archive")); assert!(!has_plugins_clone_dirs(tmp.path())); @@ -391,37 +502,141 @@ async fn sync_openai_plugins_repo_skips_archive_download_when_sha_matches() { std::fs::write(tmp.path().join(".tmp/plugins.sha"), format!("{sha}\n")).expect("write sha"); let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/repos/openai/plugins")) - .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"default_branch":"main"}"#)) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/repos/openai/plugins/git/ref/heads/main")) - .respond_with( - ResponseTemplate::new(200) - .set_body_string(format!(r#"{{"object":{{"sha":"{sha}"}}}}"#)), - ) - .mount(&server) - .await; + mount_github_repo_and_ref(&server, sha).await; - let server_uri = server.uri(); - let tmp_path = tmp.path().to_path_buf(); - tokio::task::spawn_blocking(move || { - sync_openai_plugins_repo_with_transport_overrides( - tmp_path.as_path(), - "missing-git-for-test", - &server_uri, - ) - }) + run_sync_with_transport_overrides( + tmp.path().to_path_buf(), + "missing-git-for-test", + server.uri(), + "http://127.0.0.1:9/backend-api/plugins/export/curated", + ) .await - .expect("sync task should join") .expect("sync should succeed"); assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); assert!(repo_path.join(".agents/plugins/marketplace.json").is_file()); } +#[tokio::test] +async fn sync_openai_plugins_repo_falls_back_to_export_archive_when_no_snapshot_exists() { + let tmp = tempdir().expect("tempdir"); + let server = MockServer::start().await; + let export_sha = "1111111111111111111111111111111111111111"; + + Mock::given(method("GET")) + .and(path("/repos/openai/plugins")) + .respond_with(ResponseTemplate::new(500).set_body_string("github repo lookup failed")) + .mount(&server) + .await; + let export_api_url = + mount_export_archive(&server, curated_repo_backup_archive_zip_bytes(export_sha)).await; + + let synced_sha = run_sync_with_transport_overrides( + tmp.path().to_path_buf(), + "missing-git-for-test", + server.uri(), + export_api_url, + ) + .await + .expect("export fallback sync should succeed"); + + let repo_path = curated_plugins_repo_path(tmp.path()); + assert_eq!(synced_sha, export_sha); + assert_curated_gmail_repo(&repo_path); + assert_eq!( + read_curated_plugins_sha(tmp.path()).as_deref(), + Some(export_sha) + ); +} + +#[tokio::test] +async fn sync_openai_plugins_repo_skips_export_archive_when_snapshot_exists() { + let tmp = tempdir().expect("tempdir"); + let curated_root = curated_plugins_repo_path(tmp.path()); + write_openai_curated_marketplace(&curated_root, &["linear"]); + write_curated_plugin_sha(tmp.path()); + + let plugin_manifest_path = curated_root.join("plugins/linear/.codex-plugin/plugin.json"); + let original_manifest = + std::fs::read_to_string(&plugin_manifest_path).expect("read existing plugin manifest"); + + let server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/repos/openai/plugins")) + .respond_with(ResponseTemplate::new(500).set_body_string("github repo lookup failed")) + .mount(&server) + .await; + let export_api_url = mount_export_archive( + &server, + curated_repo_backup_archive_zip_bytes("2222222222222222222222222222222222222222"), + ) + .await; + + let err = run_sync_with_transport_overrides( + tmp.path().to_path_buf(), + "missing-git-for-test", + server.uri(), + export_api_url, + ) + .await + .expect_err("existing snapshot should suppress export fallback"); + + assert!(err.contains("export archive fallback skipped")); + assert_eq!( + std::fs::read_to_string(&plugin_manifest_path).expect("read plugin manifest after sync"), + original_manifest + ); + assert_eq!( + read_curated_plugins_sha(tmp.path()).as_deref(), + Some(TEST_CURATED_PLUGIN_SHA) + ); +} + +#[test] +fn read_extracted_backup_archive_git_sha_reads_head_ref_from_extracted_repo() { + let tmp = tempdir().expect("tempdir"); + let git_dir = tmp.path().join(".git/refs/heads"); + std::fs::create_dir_all(&git_dir).expect("create git ref dir"); + std::fs::write(tmp.path().join(".git/HEAD"), "ref: refs/heads/main\n").expect("write HEAD"); + std::fs::write( + git_dir.join("main"), + "3333333333333333333333333333333333333333\n", + ) + .expect("write main ref"); + + assert_eq!( + read_extracted_backup_archive_git_sha(tmp.path()) + .expect("read extracted backup archive git sha"), + Some("3333333333333333333333333333333333333333".to_string()) + ); +} + +#[test] +fn read_extracted_backup_archive_git_sha_rejects_non_refs_head_target() { + let tmp = tempdir().expect("tempdir"); + std::fs::create_dir_all(tmp.path().join(".git")).expect("create git dir"); + std::fs::write(tmp.path().join(".git/HEAD"), "ref: HEAD\n").expect("write HEAD"); + + let err = read_extracted_backup_archive_git_sha(tmp.path()) + .expect_err("non-refs target should be rejected"); + + assert!(err.contains("must stay under refs/")); +} + +#[test] +fn read_extracted_backup_archive_git_sha_rejects_path_traversal_ref() { + let tmp = tempdir().expect("tempdir"); + std::fs::create_dir_all(tmp.path().join(".git")).expect("create git dir"); + std::fs::write(tmp.path().join(".git/HEAD"), "ref: refs/heads/../../evil\n") + .expect("write HEAD"); + + let err = read_extracted_backup_archive_git_sha(tmp.path()) + .expect_err("path traversal ref should be rejected"); + + assert!(err.contains("invalid path components")); +} + #[tokio::test] async fn startup_remote_plugin_sync_writes_marker_and_reconciles_state() { let tmp = tempdir().expect("tempdir"); @@ -528,3 +743,49 @@ fn curated_repo_zipball_bytes(sha: &str) -> Vec { writer.finish().expect("finish zip writer").into_inner() } + +fn curated_repo_backup_archive_zip_bytes(sha: &str) -> Vec { + let cursor = std::io::Cursor::new(Vec::new()); + let mut writer = ZipWriter::new(cursor); + let options = SimpleFileOptions::default(); + + writer + .start_file("plugins/.git/HEAD", options) + .expect("start HEAD entry"); + writer + .write_all(b"ref: refs/heads/main\n") + .expect("write HEAD"); + writer + .start_file("plugins/.git/refs/heads/main", options) + .expect("start main ref entry"); + writer + .write_all(format!("{sha}\n").as_bytes()) + .expect("write main ref"); + writer + .start_file("plugins/.agents/plugins/marketplace.json", options) + .expect("start marketplace entry"); + writer + .write_all( + br#"{ + "name": "openai-curated", + "plugins": [ + { + "name": "gmail", + "source": { + "source": "local", + "path": "./plugins/gmail" + } + } + ] +}"#, + ) + .expect("write marketplace"); + writer + .start_file("plugins/plugins/gmail/.codex-plugin/plugin.json", options) + .expect("start plugin manifest entry"); + writer + .write_all(br#"{"name":"gmail"}"#) + .expect("write plugin manifest"); + + writer.finish().expect("finish zip writer").into_inner() +}