feat: fallback curated plugin download from backend endpint. (#16947)

Add one more fallback for downloading the curated plugin repo from chatgpt.com. Have to be the last fallback for now as it is a lagging backup.
2026-07-01 00:31:56 +08:00 · 2026-04-06 15:36:20 -07:00
parent 36cd163504
commit 03edd4fbee
3 changed files with 691 additions and 185 deletions
@@ -24,16 +24,20 @@ use super::PluginsManager;
 const GITHUB_API_BASE_URL: &str = "https://api.github.com";
 const GITHUB_API_ACCEPT_HEADER: &str = "application/vnd.github+json";
 const GITHUB_API_VERSION_HEADER: &str = "2022-11-28";
+const CURATED_PLUGINS_BACKUP_ARCHIVE_API_URL: &str =
+    "https://chatgpt.com/backend-api/plugins/export/curated";
 const OPENAI_PLUGINS_OWNER: &str = "openai";
 const OPENAI_PLUGINS_REPO: &str = "plugins";
 const CURATED_PLUGINS_RELATIVE_DIR: &str = ".tmp/plugins";
 const CURATED_PLUGINS_SHA_FILE: &str = ".tmp/plugins.sha";
+const CURATED_PLUGINS_BACKUP_ARCHIVE_FALLBACK_VERSION: &str = "export-backup";
 const CURATED_PLUGINS_GIT_TIMEOUT: Duration = Duration::from_secs(30);
 const CURATED_PLUGINS_HTTP_TIMEOUT: Duration = Duration::from_secs(30);
+const CURATED_PLUGINS_BACKUP_ARCHIVE_TIMEOUT: Duration = Duration::from_secs(30);
 // Keep this comfortably above a normal sync attempt so we do not race another Codex process.
 const CURATED_PLUGINS_STALE_TEMP_DIR_MAX_AGE: Duration = Duration::from_secs(10 * 60);
 const STARTUP_REMOTE_PLUGIN_SYNC_MARKER_FILE: &str = ".tmp/app-server-remote-plugin-sync-v1";
-const STARTUP_REMOTE_PLUGIN_SYNC_PREREQUISITE_TIMEOUT: Duration = Duration::from_secs(5);
+const STARTUP_REMOTE_PLUGIN_SYNC_PREREQUISITE_TIMEOUT: Duration = Duration::from_secs(10);

 #[derive(Debug, Deserialize)]
 struct GitHubRepositorySummary {
@@ -50,22 +54,37 @@ struct GitHubGitRefObject {
    sha: String,
 }

+#[derive(Debug, Deserialize)]
+struct CuratedPluginsBackupArchiveResponse {
+    download_url: String,
+}
+
 pub(crate) fn curated_plugins_repo_path(codex_home: &Path) -> PathBuf {
    codex_home.join(CURATED_PLUGINS_RELATIVE_DIR)
 }

 pub(crate) fn read_curated_plugins_sha(codex_home: &Path) -> Option<String> {
-    read_sha_file(codex_home.join(CURATED_PLUGINS_SHA_FILE).as_path())
+    read_sha_file(curated_plugins_sha_path(codex_home).as_path())
+}
+
+fn curated_plugins_sha_path(codex_home: &Path) -> PathBuf {
+    codex_home.join(CURATED_PLUGINS_SHA_FILE)
 }

 pub(crate) fn sync_openai_plugins_repo(codex_home: &Path) -> Result<String, String> {
-    sync_openai_plugins_repo_with_transport_overrides(codex_home, "git", GITHUB_API_BASE_URL)
+    sync_openai_plugins_repo_with_transport_overrides(
+        codex_home,
+        "git",
+        GITHUB_API_BASE_URL,
+        CURATED_PLUGINS_BACKUP_ARCHIVE_API_URL,
+    )
 }

 fn sync_openai_plugins_repo_with_transport_overrides(
    codex_home: &Path,
    git_binary: &str,
    api_base_url: &str,
+    backup_archive_api_url: &str,
 ) -> Result<String, String> {
    match sync_openai_plugins_repo_via_git(codex_home, git_binary) {
        Ok(remote_sha) => {
@@ -80,11 +99,46 @@ fn sync_openai_plugins_repo_with_transport_overrides(
                git_binary,
                "git sync failed for curated plugin sync; falling back to GitHub HTTP"
            );
-            let result = sync_openai_plugins_repo_via_http(codex_home, api_base_url);
-            let status = if result.is_ok() { "success" } else { "failure" };
-            emit_curated_plugins_startup_sync_metric("http", status);
-            emit_curated_plugins_startup_sync_final_metric("http", status);
-            result
+            match sync_openai_plugins_repo_via_http(codex_home, api_base_url) {
+                Ok(remote_sha) => {
+                    emit_curated_plugins_startup_sync_metric("http", "success");
+                    emit_curated_plugins_startup_sync_final_metric("http", "success");
+                    Ok(remote_sha)
+                }
+                Err(http_err) => {
+                    emit_curated_plugins_startup_sync_metric("http", "failure");
+                    if has_local_curated_plugins_snapshot(codex_home) {
+                        emit_curated_plugins_startup_sync_final_metric("http", "failure");
+                        warn!(
+                            error = %http_err,
+                            "GitHub HTTP sync failed for curated plugin sync; skipping export archive fallback because a local curated plugins snapshot already exists"
+                        );
+                        Err(format!(
+                            "git sync failed for curated plugin sync: {err}; GitHub HTTP sync failed for curated plugin sync: {http_err}; export archive fallback skipped because a local curated plugins snapshot already exists"
+                        ))
+                    } else {
+                        // The export archive is a lagging backup path. Only use it to bootstrap a
+                        // missing local curated snapshot, never to refresh an existing one.
+                        warn!(
+                            error = %http_err,
+                            backup_archive_api_url,
+                            "GitHub HTTP sync failed for curated plugin sync; falling back to export archive"
+                        );
+                        let result = sync_openai_plugins_repo_via_backup_archive(
+                            codex_home,
+                            backup_archive_api_url,
+                        );
+                        let status = if result.is_ok() { "success" } else { "failure" };
+                        emit_curated_plugins_startup_sync_metric("export_archive", status);
+                        emit_curated_plugins_startup_sync_final_metric("export_archive", status);
+                        result.map_err(|export_err| {
+                            format!(
+                                "git sync failed for curated plugin sync: {err}; GitHub HTTP sync failed for curated plugin sync: {http_err}; export archive sync failed for curated plugin sync: {export_err}"
+                            )
+                        })
+                    }
+                }
+            }
        }
    }
 }
@@ -152,6 +206,29 @@ fn sync_openai_plugins_repo_via_http(
    Ok(remote_sha)
 }

+fn sync_openai_plugins_repo_via_backup_archive(
+    codex_home: &Path,
+    backup_archive_api_url: &str,
+) -> Result<String, String> {
+    let repo_path = curated_plugins_repo_path(codex_home);
+    let sha_path = curated_plugins_sha_path(codex_home);
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .map_err(|err| format!("failed to create curated plugins sync runtime: {err}"))?;
+    let staged_repo_dir = prepare_curated_repo_parent_and_temp_dir(&repo_path)?;
+    let zipball_bytes = runtime.block_on(fetch_curated_repo_backup_archive_zip(
+        backup_archive_api_url,
+    ))?;
+    extract_zipball_to_dir(&zipball_bytes, staged_repo_dir.path())?;
+    ensure_marketplace_manifest_exists(staged_repo_dir.path())?;
+    let export_version = read_extracted_backup_archive_git_sha(staged_repo_dir.path())?
+        .unwrap_or_else(|| CURATED_PLUGINS_BACKUP_ARCHIVE_FALLBACK_VERSION.to_string());
+    activate_curated_repo(&repo_path, staged_repo_dir)?;
+    write_curated_plugins_sha(&sha_path, &export_version)?;
+    Ok(export_version)
+}
+
 pub(super) fn start_startup_remote_plugin_sync_once(
    manager: Arc<PluginsManager>,
    codex_home: PathBuf,
@@ -213,17 +290,17 @@ fn startup_remote_plugin_sync_marker_path(codex_home: &Path) -> PathBuf {
    codex_home.join(STARTUP_REMOTE_PLUGIN_SYNC_MARKER_FILE)
 }

-fn startup_remote_plugin_sync_prerequisites_ready(codex_home: &Path) -> bool {
-    codex_home
-        .join(".tmp/plugins/.agents/plugins/marketplace.json")
+fn has_local_curated_plugins_snapshot(codex_home: &Path) -> bool {
+    curated_plugins_repo_path(codex_home)
+        .join(".agents/plugins/marketplace.json")
        .is_file()
-        && codex_home.join(".tmp/plugins.sha").is_file()
+        && codex_home.join(CURATED_PLUGINS_SHA_FILE).is_file()
 }

 async fn wait_for_startup_remote_plugin_sync_prerequisites(codex_home: &Path) -> bool {
    let deadline = tokio::time::Instant::now() + STARTUP_REMOTE_PLUGIN_SYNC_PREREQUISITE_TIMEOUT;
    loop {
-        if startup_remote_plugin_sync_prerequisites_ready(codex_home) {
+        if has_local_curated_plugins_snapshot(codex_home) {
            return true;
        }
        if tokio::time::Instant::now() >= deadline {
@@ -641,6 +718,126 @@ async fn fetch_curated_repo_zipball(
    fetch_github_bytes(&client, &zipball_url, "download curated plugins archive").await
 }

+async fn fetch_curated_repo_backup_archive_zip(
+    backup_archive_api_url: &str,
+) -> Result<Vec<u8>, String> {
+    let client = build_reqwest_client();
+    let export_body = fetch_public_text(
+        &client,
+        backup_archive_api_url,
+        "get curated plugins export archive metadata",
+    )
+    .await?;
+    let export_response: CuratedPluginsBackupArchiveResponse = serde_json::from_str(&export_body)
+        .map_err(|err| {
+            format!(
+                "failed to parse curated plugins backup archive response from {backup_archive_api_url}: {err}"
+            )
+        })?;
+    if export_response.download_url.is_empty() {
+        return Err(format!(
+            "curated plugins backup archive response from {backup_archive_api_url} did not include a download URL"
+        ));
+    }
+
+    fetch_public_bytes(
+        &client,
+        &export_response.download_url,
+        "download curated plugins export archive",
+    )
+    .await
+}
+
+fn read_extracted_backup_archive_git_sha(repo_path: &Path) -> Result<Option<String>, String> {
+    let git_dir = repo_path.join(".git");
+    if !git_dir.is_dir() {
+        return Ok(None);
+    }
+
+    let head_path = git_dir.join("HEAD");
+    let head = std::fs::read_to_string(&head_path).map_err(|err| {
+        format!(
+            "failed to read curated plugins backup archive git HEAD {}: {err}",
+            head_path.display()
+        )
+    })?;
+    let head = head.trim();
+    if head.is_empty() {
+        return Err(format!(
+            "curated plugins backup archive git HEAD is empty at {}",
+            head_path.display()
+        ));
+    }
+
+    if let Some(reference) = head.strip_prefix("ref: ") {
+        let reference = validate_backup_archive_git_ref(reference.trim())?;
+        return read_git_ref_sha(&git_dir, reference).map(Some);
+    }
+
+    Ok(Some(head.to_string()))
+}
+
+fn validate_backup_archive_git_ref(reference: &str) -> Result<&str, String> {
+    if !reference.starts_with("refs/") {
+        return Err(format!(
+            "curated plugins backup archive git ref must stay under refs/: {reference}"
+        ));
+    }
+
+    let path = Path::new(reference);
+    if path.is_absolute() {
+        return Err(format!(
+            "curated plugins backup archive git ref must be relative: {reference}"
+        ));
+    }
+
+    for component in path.components() {
+        match component {
+            std::path::Component::Normal(_) => {}
+            _ => {
+                return Err(format!(
+                    "curated plugins backup archive git ref contains invalid path components: {reference}"
+                ));
+            }
+        }
+    }
+
+    Ok(reference)
+}
+
+fn read_git_ref_sha(git_dir: &Path, reference: &str) -> Result<String, String> {
+    let ref_path = git_dir.join(reference);
+    if let Ok(sha) = std::fs::read_to_string(&ref_path) {
+        let sha = sha.trim();
+        if sha.is_empty() {
+            return Err(format!(
+                "curated plugins backup archive git ref {reference} is empty at {}",
+                ref_path.display()
+            ));
+        }
+        return Ok(sha.to_string());
+    }
+
+    let packed_refs_path = git_dir.join("packed-refs");
+    if let Ok(packed_refs) = std::fs::read_to_string(&packed_refs_path)
+        && let Some(sha) = packed_refs.lines().find_map(|line| {
+            let trimmed = line.trim();
+            if trimmed.is_empty() || trimmed.starts_with('#') || trimmed.starts_with('^') {
+                return None;
+            }
+            let (sha, candidate_ref) = trimmed.split_once(' ')?;
+            (candidate_ref == reference).then_some(sha.to_string())
+        })
+    {
+        return Ok(sha);
+    }
+
+    Err(format!(
+        "failed to resolve curated plugins backup archive git ref {reference} from {}",
+        git_dir.display()
+    ))
+}
+
 async fn fetch_github_text(client: &Client, url: &str, context: &str) -> Result<String, String> {
    let response = github_request(client, url)
        .send()
@@ -675,6 +872,44 @@ async fn fetch_github_bytes(client: &Client, url: &str, context: &str) -> Result
    Ok(body.to_vec())
 }

+async fn fetch_public_text(client: &Client, url: &str, context: &str) -> Result<String, String> {
+    let response = client
+        .get(url)
+        .timeout(CURATED_PLUGINS_BACKUP_ARCHIVE_TIMEOUT)
+        .send()
+        .await
+        .map_err(|err| format!("failed to {context} from {url}: {err}"))?;
+    let status = response.status();
+    let body = response.text().await.unwrap_or_default();
+    if !status.is_success() {
+        return Err(format!(
+            "{context} from {url} failed with status {status}: {body}"
+        ));
+    }
+    Ok(body)
+}
+
+async fn fetch_public_bytes(client: &Client, url: &str, context: &str) -> Result<Vec<u8>, String> {
+    let response = client
+        .get(url)
+        .timeout(CURATED_PLUGINS_BACKUP_ARCHIVE_TIMEOUT)
+        .send()
+        .await
+        .map_err(|err| format!("failed to {context} from {url}: {err}"))?;
+    let status = response.status();
+    let body = response
+        .bytes()
+        .await
+        .map_err(|err| format!("failed to read {context} response from {url}: {err}"))?;
+    if !status.is_success() {
+        let body_text = String::from_utf8_lossy(&body);
+        return Err(format!(
+            "{context} from {url} failed with status {status}: {body_text}"
+        ));
+    }
+    Ok(body.to_vec())
+}
+
 fn github_request(client: &Client, url: &str) -> reqwest::RequestBuilder {
    client
        .get(url)