Auto-upgrade configured marketplaces (#17425)

## Summary
- Add best-effort auto-upgrade for user-configured Git marketplaces
recorded in `config.toml`.
- Track the last activated Git revision with `last_revision` so
unchanged marketplace sources skip clone work.
- Trigger the upgrade from plugin startup and `plugin/list`, while
preserving existing fail-open plugin behavior with warning logs rather
than new user-visible errors.

## Details
- Remote configured marketplaces use `git ls-remote` to compare the
source/ref against the recorded revision.
- Upgrades clone into a staging directory, validate that
`.agents/plugins/marketplace.json` exists and that the manifest name
matches the configured marketplace key, then atomically activate the new
root.
- Local `.agents/plugins/marketplace.json` marketplaces remain live
filesystem state and are not auto-pulled.
- Existing non-curated plugin cache refresh is kicked after successful
marketplace root upgrades.

## Validation
- `just write-config-schema`
- `cargo test -p codex-core marketplace_upgrade`
- `cargo check -p codex-cli -p codex-app-server`
- `just fix -p codex-core`

Did not run the complete `cargo test` suite because the repo
instructions require asking before a full core workspace run.
This commit is contained in:
xli-oai
2026-04-16 10:36:34 -07:00
committed by GitHub
Unverified
parent 109b22a8d0
commit faf48489f3
17 changed files with 1094 additions and 34 deletions
+1
View File
@@ -2019,6 +2019,7 @@ dependencies = [
name = "codex-core-plugins"
version = "0.0.0"
dependencies = [
"chrono",
"codex-app-server-protocol",
"codex-config",
"codex-core-skills",
@@ -6178,7 +6178,7 @@ impl CodexMessageProcessor {
force_remote_sync,
} = params;
let roots = cwds.unwrap_or_default();
plugins_manager.maybe_start_non_curated_plugin_cache_refresh_for_roots(&roots);
plugins_manager.maybe_start_non_curated_plugin_cache_refresh(&roots);
let mut config = match self.load_latest_config(/*fallback_cwd*/ None).await {
Ok(config) => config,
+75 -6
View File
@@ -1,8 +1,12 @@
use anyhow::Context;
use anyhow::Result;
use anyhow::bail;
use clap::Parser;
use codex_core::config::Config;
use codex_core::config::find_codex_home;
use codex_core::plugins::MarketplaceAddRequest;
use codex_core::plugins::PluginMarketplaceUpgradeOutcome;
use codex_core::plugins::PluginsManager;
use codex_core::plugins::add_marketplace;
use codex_utils_cli::CliConfigOverrides;
@@ -17,8 +21,8 @@ pub struct MarketplaceCli {
#[derive(Debug, clap::Subcommand)]
enum MarketplaceSubcommand {
/// Add a remote marketplace repository.
Add(AddMarketplaceArgs),
Upgrade(UpgradeMarketplaceArgs),
}
#[derive(Debug, Parser)]
@@ -27,11 +31,9 @@ struct AddMarketplaceArgs {
/// or local marketplace root directories.
source: String,
/// Git ref to check out. Overrides any @ref or #ref suffix in SOURCE.
#[arg(long = "ref", value_name = "REF")]
ref_name: Option<String>,
/// Sparse-checkout path to use while cloning git sources. Repeat to include multiple paths.
#[arg(
long = "sparse",
value_name = "PATH",
@@ -40,6 +42,11 @@ struct AddMarketplaceArgs {
sparse_paths: Vec<String>,
}
#[derive(Debug, Parser)]
struct UpgradeMarketplaceArgs {
marketplace_name: Option<String>,
}
impl MarketplaceCli {
pub async fn run(self) -> Result<()> {
let MarketplaceCli {
@@ -47,14 +54,13 @@ impl MarketplaceCli {
subcommand,
} = self;
// Validate overrides now. This command writes to CODEX_HOME only; marketplace discovery
// happens from that cache root after the next plugin/list or app-server start.
config_overrides
let overrides = config_overrides
.parse_overrides()
.map_err(anyhow::Error::msg)?;
match subcommand {
MarketplaceSubcommand::Add(args) => run_add(args).await?,
MarketplaceSubcommand::Upgrade(args) => run_upgrade(overrides, args).await?,
}
Ok(())
@@ -98,6 +104,60 @@ async fn run_add(args: AddMarketplaceArgs) -> Result<()> {
Ok(())
}
async fn run_upgrade(
overrides: Vec<(String, toml::Value)>,
args: UpgradeMarketplaceArgs,
) -> Result<()> {
let UpgradeMarketplaceArgs { marketplace_name } = args;
let config = Config::load_with_cli_overrides(overrides)
.await
.context("failed to load configuration")?;
let codex_home = find_codex_home().context("failed to resolve CODEX_HOME")?;
let manager = PluginsManager::new(codex_home.to_path_buf());
let outcome = manager
.upgrade_configured_marketplaces_for_config(&config, marketplace_name.as_deref())
.map_err(anyhow::Error::msg)?;
print_upgrade_outcome(&outcome, marketplace_name.as_deref())
}
fn print_upgrade_outcome(
outcome: &PluginMarketplaceUpgradeOutcome,
marketplace_name: Option<&str>,
) -> Result<()> {
for error in &outcome.errors {
eprintln!(
"Failed to upgrade marketplace `{}`: {}",
error.marketplace_name, error.message
);
}
if !outcome.all_succeeded() {
bail!("{} upgrade failure(s) occurred.", outcome.errors.len());
}
let selection_label = marketplace_name.unwrap_or("all configured Git marketplaces");
if outcome.selected_marketplaces.is_empty() {
println!("No configured Git marketplaces to upgrade.");
} else if outcome.upgraded_roots.is_empty() {
if marketplace_name.is_some() {
println!("Marketplace `{selection_label}` is already up to date.");
} else {
println!("All configured Git marketplaces are already up to date.");
}
} else if marketplace_name.is_some() {
println!("Upgraded marketplace `{selection_label}` to the latest configured revision.");
for root in &outcome.upgraded_roots {
println!("Installed marketplace root: {}", root.display());
}
} else {
println!("Upgraded {} marketplace(s).", outcome.upgraded_roots.len());
for root in &outcome.upgraded_roots {
println!("Installed marketplace root: {}", root.display());
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
@@ -132,4 +192,13 @@ mod tests {
vec!["plugins/foo", "skills/bar"]
);
}
#[test]
fn upgrade_subcommand_parses_optional_marketplace_name() {
let upgrade_all = UpgradeMarketplaceArgs::try_parse_from(["upgrade"]).unwrap();
assert_eq!(upgrade_all.marketplace_name, None);
let upgrade_one = UpgradeMarketplaceArgs::try_parse_from(["upgrade", "debug"]).unwrap();
assert_eq!(upgrade_one.marketplace_name.as_deref(), Some("debug"));
}
}
+4
View File
@@ -12,6 +12,7 @@ use crate::CONFIG_TOML_FILE;
pub struct MarketplaceConfigUpdate<'a> {
pub last_updated: &'a str,
pub last_revision: Option<&'a str>,
pub source_type: &'a str,
pub source: &'a str,
pub ref_name: Option<&'a str>,
@@ -63,6 +64,9 @@ fn upsert_marketplace(
let mut entry = TomlTable::new();
entry.set_implicit(false);
entry["last_updated"] = value(update.last_updated.to_string());
if let Some(last_revision) = update.last_revision {
entry["last_revision"] = value(last_revision.to_string());
}
entry["source_type"] = value(update.source_type.to_string());
entry["source"] = value(update.source.to_string());
if let Some(ref_name) = update.ref_name {
+3
View File
@@ -626,6 +626,9 @@ pub struct MarketplaceConfig {
/// Last time Codex successfully added or refreshed this marketplace.
#[serde(default)]
pub last_updated: Option<String>,
/// Git revision Codex last successfully activated for this marketplace.
#[serde(default)]
pub last_revision: Option<String>,
/// Source kind used to install this marketplace.
#[serde(default)]
pub source_type: Option<MarketplaceSourceType>,
+1
View File
@@ -23,6 +23,7 @@ codex-plugin = { workspace = true }
codex-protocol = { workspace = true }
codex-utils-absolute-path = { workspace = true }
codex-utils-plugins = { workspace = true }
chrono = { workspace = true }
dirs = { workspace = true }
reqwest = { workspace = true }
serde = { workspace = true, features = ["derive"] }
+1
View File
@@ -1,6 +1,7 @@
pub mod loader;
pub mod manifest;
pub mod marketplace;
pub mod marketplace_upgrade;
pub mod remote;
pub mod store;
pub mod toggles;
+32 -1
View File
@@ -41,6 +41,12 @@ const DEFAULT_APP_CONFIG_FILE: &str = ".app.json";
const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated";
const CONFIG_TOML_FILE: &str = "config.toml";
#[derive(Clone, Copy, PartialEq, Eq)]
enum NonCuratedCacheRefreshMode {
IfVersionChanged,
ForceReinstall,
}
pub fn log_plugin_load_errors(outcome: &PluginLoadOutcome<McpServerConfig>) {
for plugin in outcome
.plugins()
@@ -180,6 +186,29 @@ pub fn refresh_curated_plugin_cache(
pub fn refresh_non_curated_plugin_cache(
codex_home: &Path,
additional_roots: &[AbsolutePathBuf],
) -> Result<bool, String> {
refresh_non_curated_plugin_cache_with_mode(
codex_home,
additional_roots,
NonCuratedCacheRefreshMode::IfVersionChanged,
)
}
pub fn refresh_non_curated_plugin_cache_force_reinstall(
codex_home: &Path,
additional_roots: &[AbsolutePathBuf],
) -> Result<bool, String> {
refresh_non_curated_plugin_cache_with_mode(
codex_home,
additional_roots,
NonCuratedCacheRefreshMode::ForceReinstall,
)
}
fn refresh_non_curated_plugin_cache_with_mode(
codex_home: &Path,
additional_roots: &[AbsolutePathBuf],
mode: NonCuratedCacheRefreshMode,
) -> Result<bool, String> {
let configured_non_curated_plugin_ids =
non_curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home(
@@ -248,7 +277,9 @@ pub fn refresh_non_curated_plugin_cache(
continue;
};
if store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str()) {
if mode == NonCuratedCacheRefreshMode::IfVersionChanged
&& store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str())
{
continue;
}
@@ -0,0 +1,298 @@
mod activation;
mod git;
use self::activation::activate_marketplace_root;
use self::activation::installed_marketplace_metadata_matches;
use self::activation::write_installed_marketplace_metadata;
use self::git::clone_git_source;
use self::git::git_remote_revision;
use crate::marketplace::validate_marketplace_root;
use codex_config::CONFIG_TOML_FILE;
use codex_config::ConfigLayerStack;
use codex_config::MarketplaceConfigUpdate;
use codex_config::record_user_marketplace;
use codex_config::types::MarketplaceConfig;
use codex_config::types::MarketplaceSourceType;
use codex_plugin::validate_plugin_segment;
use codex_utils_absolute_path::AbsolutePathBuf;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
use std::time::Duration;
use tracing::warn;
const INSTALLED_MARKETPLACES_DIR: &str = ".tmp/marketplaces";
const MARKETPLACE_UPGRADE_GIT_TIMEOUT: Duration = Duration::from_secs(30);
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ConfiguredMarketplaceUpgradeError {
pub marketplace_name: String,
pub message: String,
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct ConfiguredMarketplaceUpgradeOutcome {
pub selected_marketplaces: Vec<String>,
pub upgraded_roots: Vec<AbsolutePathBuf>,
pub errors: Vec<ConfiguredMarketplaceUpgradeError>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct ConfiguredGitMarketplace {
name: String,
source: String,
ref_name: Option<String>,
sparse_paths: Vec<String>,
last_revision: Option<String>,
}
impl ConfiguredMarketplaceUpgradeOutcome {
pub fn all_succeeded(&self) -> bool {
self.errors.is_empty()
}
}
pub fn configured_git_marketplace_names(config_layer_stack: &ConfigLayerStack) -> Vec<String> {
let mut names = configured_git_marketplaces(config_layer_stack)
.into_iter()
.map(|marketplace| marketplace.name)
.collect::<Vec<_>>();
names.sort_unstable();
names
}
pub fn upgrade_configured_git_marketplaces(
codex_home: &Path,
config_layer_stack: &ConfigLayerStack,
marketplace_name: Option<&str>,
) -> ConfiguredMarketplaceUpgradeOutcome {
let marketplaces = configured_git_marketplaces(config_layer_stack)
.into_iter()
.filter(|marketplace| marketplace_name.is_none_or(|name| marketplace.name.as_str() == name))
.collect::<Vec<_>>();
if marketplaces.is_empty() {
return ConfiguredMarketplaceUpgradeOutcome::default();
}
let install_root = marketplace_install_root(codex_home);
let selected_marketplaces = marketplaces
.iter()
.map(|marketplace| marketplace.name.clone())
.collect();
let mut upgraded_roots = Vec::new();
let mut errors = Vec::new();
for marketplace in marketplaces {
match upgrade_configured_git_marketplace(codex_home, &install_root, &marketplace) {
Ok(Some(upgraded_root)) => upgraded_roots.push(upgraded_root),
Ok(None) => {}
Err(err) => {
errors.push(ConfiguredMarketplaceUpgradeError {
marketplace_name: marketplace.name,
message: err,
});
}
}
}
ConfiguredMarketplaceUpgradeOutcome {
selected_marketplaces,
upgraded_roots,
errors,
}
}
fn marketplace_install_root(codex_home: &Path) -> PathBuf {
codex_home.join(INSTALLED_MARKETPLACES_DIR)
}
fn configured_git_marketplaces(
config_layer_stack: &ConfigLayerStack,
) -> Vec<ConfiguredGitMarketplace> {
let Some(user_layer) = config_layer_stack.get_user_layer() else {
return Vec::new();
};
let Some(marketplaces_value) = user_layer.config.get("marketplaces") else {
return Vec::new();
};
let marketplaces = match marketplaces_value
.clone()
.try_into::<HashMap<String, MarketplaceConfig>>()
{
Ok(marketplaces) => marketplaces,
Err(err) => {
warn!("invalid marketplaces config while preparing auto-upgrade: {err}");
return Vec::new();
}
};
let mut configured = marketplaces
.into_iter()
.filter_map(|(name, marketplace)| configured_git_marketplace_from_config(name, marketplace))
.collect::<Vec<_>>();
configured.sort_unstable_by(|left, right| left.name.cmp(&right.name));
configured
}
fn configured_git_marketplace_from_config(
name: String,
marketplace: MarketplaceConfig,
) -> Option<ConfiguredGitMarketplace> {
let MarketplaceConfig {
last_updated: _,
last_revision,
source_type,
source,
ref_name,
sparse_paths,
} = marketplace;
if source_type != Some(MarketplaceSourceType::Git) {
return None;
}
let Some(source) = source else {
warn!(
marketplace = name,
"ignoring configured Git marketplace without source"
);
return None;
};
Some(ConfiguredGitMarketplace {
name,
source,
ref_name,
sparse_paths: sparse_paths.unwrap_or_default(),
last_revision,
})
}
fn upgrade_configured_git_marketplace(
codex_home: &Path,
install_root: &Path,
marketplace: &ConfiguredGitMarketplace,
) -> Result<Option<AbsolutePathBuf>, String> {
validate_plugin_segment(&marketplace.name, "marketplace name")?;
let remote_revision = git_remote_revision(
&marketplace.source,
marketplace.ref_name.as_deref(),
MARKETPLACE_UPGRADE_GIT_TIMEOUT,
)?;
let destination = install_root.join(&marketplace.name);
if destination
.join(".agents/plugins/marketplace.json")
.is_file()
&& marketplace.last_revision.as_deref() == Some(remote_revision.as_str())
&& installed_marketplace_metadata_matches(&destination, marketplace, &remote_revision)
{
return Ok(None);
}
let staging_parent = install_root.join(".staging");
std::fs::create_dir_all(&staging_parent).map_err(|err| {
format!(
"failed to create marketplace upgrade staging directory {}: {err}",
staging_parent.display()
)
})?;
let staged_dir = tempfile::Builder::new()
.prefix("marketplace-upgrade-")
.tempdir_in(&staging_parent)
.map_err(|err| {
format!(
"failed to create temporary marketplace upgrade directory in {}: {err}",
staging_parent.display()
)
})?;
let activated_revision = clone_git_source(
&marketplace.source,
marketplace.ref_name.as_deref(),
&marketplace.sparse_paths,
staged_dir.path(),
MARKETPLACE_UPGRADE_GIT_TIMEOUT,
)?;
let marketplace_name = validate_marketplace_root(staged_dir.path())
.map_err(|err| format!("failed to validate upgraded marketplace root: {err}"))?;
if marketplace_name != marketplace.name {
return Err(format!(
"upgraded marketplace name `{marketplace_name}` does not match configured marketplace `{}`",
marketplace.name
));
}
write_installed_marketplace_metadata(staged_dir.path(), marketplace, &activated_revision)?;
let last_updated = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
let update = MarketplaceConfigUpdate {
last_updated: &last_updated,
last_revision: Some(&activated_revision),
source_type: "git",
source: &marketplace.source,
ref_name: marketplace.ref_name.as_deref(),
sparse_paths: &marketplace.sparse_paths,
};
activate_marketplace_root(&destination, staged_dir, || {
ensure_configured_git_marketplace_unchanged(codex_home, marketplace)?;
record_user_marketplace(codex_home, &marketplace.name, &update).map_err(|err| {
format!(
"failed to record upgraded marketplace `{}` in user config.toml: {err}",
marketplace.name
)
})
})?;
AbsolutePathBuf::try_from(destination)
.map(Some)
.map_err(|err| format!("upgraded marketplace path is not absolute: {err}"))
}
fn ensure_configured_git_marketplace_unchanged(
codex_home: &Path,
expected: &ConfiguredGitMarketplace,
) -> Result<(), String> {
let current = read_configured_git_marketplace(codex_home, &expected.name)?;
match current {
Some(current) if current == *expected => Ok(()),
Some(_) => Err(format!(
"configured marketplace `{}` changed while auto-upgrade was in flight",
expected.name
)),
None => Err(format!(
"configured marketplace `{}` was removed or is no longer a Git marketplace",
expected.name
)),
}
}
fn read_configured_git_marketplace(
codex_home: &Path,
marketplace_name: &str,
) -> Result<Option<ConfiguredGitMarketplace>, String> {
let config_path = codex_home.join(CONFIG_TOML_FILE);
let raw_config = match std::fs::read_to_string(&config_path) {
Ok(raw_config) => raw_config,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(err) => {
return Err(format!(
"failed to read user config {} while checking marketplace auto-upgrade: {err}",
config_path.display()
));
}
};
let config: toml::Value = toml::from_str(&raw_config).map_err(|err| {
format!(
"failed to parse user config {} while checking marketplace auto-upgrade: {err}",
config_path.display()
)
})?;
let Some(marketplaces_value) = config.get("marketplaces") else {
return Ok(None);
};
let mut marketplaces = marketplaces_value
.clone()
.try_into::<HashMap<String, MarketplaceConfig>>()
.map_err(|err| format!("invalid marketplaces config while checking auto-upgrade: {err}"))?;
let Some(marketplace) = marketplaces.remove(marketplace_name) else {
return Ok(None);
};
Ok(configured_git_marketplace_from_config(
marketplace_name.to_string(),
marketplace,
))
}
@@ -0,0 +1,167 @@
use super::ConfiguredGitMarketplace;
use codex_config::types::MarketplaceSourceType;
use serde::Deserialize;
use serde::Serialize;
use std::path::Path;
use std::path::PathBuf;
use tempfile::TempDir;
use tracing::warn;
const MARKETPLACE_INSTALL_METADATA_FILE: &str = ".codex-marketplace-install.json";
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
struct InstalledMarketplaceMetadata {
source_type: MarketplaceSourceType,
source: String,
ref_name: Option<String>,
sparse_paths: Vec<String>,
revision: String,
}
pub(super) fn installed_marketplace_metadata_matches(
root: &Path,
marketplace: &ConfiguredGitMarketplace,
revision: &str,
) -> bool {
let metadata = match std::fs::read_to_string(installed_marketplace_metadata_path(root)) {
Ok(metadata) => metadata,
Err(_) => return false,
};
let metadata = match serde_json::from_str::<InstalledMarketplaceMetadata>(&metadata) {
Ok(metadata) => metadata,
Err(err) => {
warn!(
marketplace = marketplace.name,
error = %err,
"failed to parse activated marketplace metadata"
);
return false;
}
};
metadata == installed_marketplace_metadata(marketplace, revision)
}
pub(super) fn write_installed_marketplace_metadata(
root: &Path,
marketplace: &ConfiguredGitMarketplace,
revision: &str,
) -> Result<(), String> {
let metadata = installed_marketplace_metadata(marketplace, revision);
let contents = serde_json::to_string_pretty(&metadata)
.map_err(|err| format!("failed to serialize activated marketplace metadata: {err}"))?;
std::fs::write(installed_marketplace_metadata_path(root), contents)
.map_err(|err| format!("failed to write activated marketplace metadata: {err}"))
}
pub(super) fn activate_marketplace_root(
destination: &Path,
staged_dir: TempDir,
after_activate: impl FnOnce() -> Result<(), String>,
) -> Result<(), String> {
let staged_root = staged_dir.path();
let Some(parent) = destination.parent() else {
return Err(format!(
"failed to determine marketplace install parent for {}",
destination.display()
));
};
std::fs::create_dir_all(parent).map_err(|err| {
format!(
"failed to create marketplace install parent {}: {err}",
parent.display()
)
})?;
if destination.exists() {
let backup_dir = tempfile::Builder::new()
.prefix("marketplace-backup-")
.tempdir_in(parent)
.map_err(|err| {
format!(
"failed to create marketplace backup directory in {}: {err}",
parent.display()
)
})?;
let backup_root = backup_dir.path().join("root");
std::fs::rename(destination, &backup_root).map_err(|err| {
format!(
"failed to move previous marketplace root out of the way at {}: {err}",
destination.display()
)
})?;
if let Err(err) = std::fs::rename(staged_root, destination) {
let rollback_result = std::fs::rename(&backup_root, destination);
return match rollback_result {
Ok(()) => Err(format!(
"failed to activate upgraded marketplace at {}: {err}",
destination.display()
)),
Err(rollback_err) => {
let backup_path = backup_dir.keep().join("root");
Err(format!(
"failed to activate upgraded marketplace at {}: {err}; failed to restore previous marketplace root (left at {}): {rollback_err}",
destination.display(),
backup_path.display()
))
}
};
}
if let Err(err) = after_activate() {
let remove_result = std::fs::remove_dir_all(destination);
let rollback_result =
remove_result.and_then(|()| std::fs::rename(&backup_root, destination));
return match rollback_result {
Ok(()) => Err(err),
Err(rollback_err) => {
let backup_path = backup_dir.keep().join("root");
Err(format!(
"{err}; failed to restore previous marketplace root at {} (left at {}): {rollback_err}",
destination.display(),
backup_path.display()
))
}
};
}
return Ok(());
}
std::fs::rename(staged_root, destination).map_err(|err| {
format!(
"failed to activate upgraded marketplace at {}: {err}",
destination.display()
)
})?;
if let Err(err) = after_activate() {
let remove_result = std::fs::remove_dir_all(destination);
return match remove_result {
Ok(()) => Err(err),
Err(remove_err) => Err(format!(
"{err}; failed to remove newly activated marketplace root at {}: {remove_err}",
destination.display()
)),
};
}
Ok(())
}
fn installed_marketplace_metadata(
marketplace: &ConfiguredGitMarketplace,
revision: &str,
) -> InstalledMarketplaceMetadata {
InstalledMarketplaceMetadata {
source_type: MarketplaceSourceType::Git,
source: marketplace.source.clone(),
ref_name: marketplace.ref_name.clone(),
sparse_paths: marketplace.sparse_paths.clone(),
revision: revision.to_string(),
}
}
fn installed_marketplace_metadata_path(root: &Path) -> PathBuf {
root.join(MARKETPLACE_INSTALL_METADATA_FILE)
}
@@ -0,0 +1,238 @@
use std::path::Path;
use std::process::Command;
use std::process::Output;
use std::process::Stdio;
use std::time::Duration;
pub(super) fn git_remote_revision(
source: &str,
ref_name: Option<&str>,
timeout: Duration,
) -> Result<String, String> {
if let Some(ref_name) = ref_name
&& is_full_git_sha(ref_name)
{
return Ok(ref_name.to_string());
}
let ref_name = ref_name.unwrap_or("HEAD");
let output = run_git_command_with_timeout(
git_command().arg("ls-remote").arg(source).arg(ref_name),
"git ls-remote marketplace source",
timeout,
)?;
ensure_git_success(&output, "git ls-remote marketplace source")?;
let stdout = String::from_utf8_lossy(&output.stdout);
let Some(first_line) = stdout.lines().next() else {
return Err("git ls-remote returned empty output for marketplace source".to_string());
};
let Some((revision, _)) = first_line.split_once('\t') else {
return Err(format!(
"unexpected git ls-remote output for marketplace source: {first_line}"
));
};
let revision = revision.trim();
if revision.is_empty() {
return Err("git ls-remote returned empty revision for marketplace source".to_string());
}
Ok(revision.to_string())
}
pub(super) fn clone_git_source(
source: &str,
ref_name: Option<&str>,
sparse_paths: &[String],
destination: &Path,
timeout: Duration,
) -> Result<String, String> {
if sparse_paths.is_empty() {
let output = run_git_command_with_timeout(
git_command().arg("clone").arg(source).arg(destination),
"git clone marketplace source",
timeout,
)?;
ensure_git_success(&output, "git clone marketplace source")?;
if let Some(ref_name) = ref_name {
let output = run_git_command_with_timeout(
git_command()
.arg("-C")
.arg(destination)
.arg("checkout")
.arg(ref_name),
"git checkout marketplace ref",
timeout,
)?;
ensure_git_success(&output, "git checkout marketplace ref")?;
}
return git_worktree_revision(destination, timeout);
}
let output = run_git_command_with_timeout(
git_command()
.arg("clone")
.arg("--filter=blob:none")
.arg("--no-checkout")
.arg(source)
.arg(destination),
"git clone marketplace source",
timeout,
)?;
ensure_git_success(&output, "git clone marketplace source")?;
let mut sparse_checkout = git_command();
sparse_checkout
.arg("-C")
.arg(destination)
.arg("sparse-checkout")
.arg("set")
.args(sparse_paths);
let output = run_git_command_with_timeout(
&mut sparse_checkout,
"git sparse-checkout marketplace source",
timeout,
)?;
ensure_git_success(&output, "git sparse-checkout marketplace source")?;
let output = run_git_command_with_timeout(
git_command()
.arg("-C")
.arg(destination)
.arg("checkout")
.arg(ref_name.unwrap_or("HEAD")),
"git checkout marketplace ref",
timeout,
)?;
ensure_git_success(&output, "git checkout marketplace ref")?;
git_worktree_revision(destination, timeout)
}
fn git_worktree_revision(destination: &Path, timeout: Duration) -> Result<String, String> {
let output = run_git_command_with_timeout(
git_command()
.arg("-C")
.arg(destination)
.arg("rev-parse")
.arg("HEAD"),
"git rev-parse marketplace revision",
timeout,
)?;
ensure_git_success(&output, "git rev-parse marketplace revision")?;
let revision = String::from_utf8_lossy(&output.stdout).trim().to_string();
if revision.is_empty() {
Err("git rev-parse returned empty revision for marketplace source".to_string())
} else {
Ok(revision)
}
}
fn is_full_git_sha(value: &str) -> bool {
value.len() == 40 && value.chars().all(|ch| ch.is_ascii_hexdigit())
}
fn git_command() -> Command {
let mut command = Command::new("git");
command
.env("GIT_OPTIONAL_LOCKS", "0")
.env("GIT_TERMINAL_PROMPT", "0");
command
}
fn run_git_command_with_timeout(
command: &mut Command,
context: &str,
timeout: Duration,
) -> Result<Output, String> {
let mut child = command
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|err| format!("failed to run {context}: {err}"))?;
let start = std::time::Instant::now();
loop {
match child.try_wait() {
Ok(Some(_)) => {
return child
.wait_with_output()
.map_err(|err| format!("failed to wait for {context}: {err}"));
}
Ok(None) => {}
Err(err) => return Err(format!("failed to poll {context}: {err}")),
}
if start.elapsed() >= timeout {
let _ = child.kill();
let output = child
.wait_with_output()
.map_err(|err| format!("failed to wait for {context} after timeout: {err}"))?;
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
return if stderr.is_empty() {
Err(format!("{context} timed out after {}s", timeout.as_secs()))
} else {
Err(format!(
"{context} timed out after {}s: {stderr}",
timeout.as_secs()
))
};
}
std::thread::sleep(Duration::from_millis(100));
}
}
fn ensure_git_success(output: &Output, context: &str) -> Result<(), String> {
if output.status.success() {
return Ok(());
}
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
if stderr.is_empty() {
Err(format!("{context} failed with status {}", output.status))
} else {
Err(format!(
"{context} failed with status {}: {stderr}",
output.status
))
}
}
#[cfg(test)]
mod tests {
use super::git_command;
use super::is_full_git_sha;
use std::ffi::OsStr;
#[test]
fn full_git_sha_ref_is_already_a_remote_revision() {
assert!(is_full_git_sha("0123456789abcdef0123456789abcdef01234567"));
assert!(!is_full_git_sha("main"));
assert!(!is_full_git_sha("0123456"));
}
#[test]
fn git_command_uses_path_lookup_with_stable_noninteractive_env() {
let command = git_command();
assert_eq!(command.get_program(), OsStr::new("git"));
assert_eq!(
command_env(&command, "GIT_OPTIONAL_LOCKS"),
Some(Some(OsStr::new("0")))
);
assert_eq!(
command_env(&command, "GIT_TERMINAL_PROMPT"),
Some(Some(OsStr::new("0")))
);
assert_eq!(command_env(&command, "PATH"), None);
}
fn command_env<'a>(
command: &'a std::process::Command,
name: &str,
) -> Option<Option<&'a OsStr>> {
command
.get_envs()
.find(|(key, _)| key == &OsStr::new(name))
.map(|(_, value)| value)
}
}
+5
View File
@@ -779,6 +779,11 @@
"MarketplaceConfig": {
"additionalProperties": false,
"properties": {
"last_revision": {
"default": null,
"description": "Git revision Codex last successfully activated for this marketplace.",
"type": "string"
},
"last_updated": {
"default": null,
"description": "Last time Codex successfully added or refreshed this marketplace.",
+183 -26
View File
@@ -25,6 +25,7 @@ use codex_core_plugins::loader::log_plugin_load_errors;
use codex_core_plugins::loader::plugin_telemetry_metadata_from_root;
use codex_core_plugins::loader::refresh_curated_plugin_cache;
use codex_core_plugins::loader::refresh_non_curated_plugin_cache;
use codex_core_plugins::loader::refresh_non_curated_plugin_cache_force_reinstall;
use codex_core_plugins::manifest::PluginManifestInterface;
use codex_core_plugins::manifest::load_plugin_manifest;
use codex_core_plugins::marketplace::MarketplaceError;
@@ -37,6 +38,10 @@ use codex_core_plugins::marketplace::ResolvedMarketplacePlugin;
use codex_core_plugins::marketplace::list_marketplaces;
use codex_core_plugins::marketplace::load_marketplace;
use codex_core_plugins::marketplace::resolve_marketplace_plugin;
use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeError;
use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeOutcome;
use codex_core_plugins::marketplace_upgrade::configured_git_marketplace_names;
use codex_core_plugins::marketplace_upgrade::upgrade_configured_git_marketplaces;
use codex_core_plugins::remote::RemotePluginFetchError;
use codex_core_plugins::remote::RemotePluginMutationError;
use codex_core_plugins::remote::RemotePluginServiceConfig;
@@ -88,10 +93,27 @@ struct CachedFeaturedPluginIds {
featured_plugin_ids: Vec<String>,
}
#[derive(Clone, PartialEq, Eq)]
struct NonCuratedCacheRefreshRequest {
roots: Vec<AbsolutePathBuf>,
mode: NonCuratedCacheRefreshMode,
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum NonCuratedCacheRefreshMode {
IfVersionChanged,
ForceReinstall,
}
#[derive(Default)]
struct NonCuratedCacheRefreshState {
requested_roots: Option<Vec<AbsolutePathBuf>>,
last_refreshed_roots: Option<Vec<AbsolutePathBuf>>,
requested: Option<NonCuratedCacheRefreshRequest>,
last_refreshed: Option<NonCuratedCacheRefreshRequest>,
in_flight: bool,
}
#[derive(Default)]
struct ConfiguredMarketplaceUpgradeState {
in_flight: bool,
}
@@ -314,6 +336,7 @@ pub struct PluginsManager {
codex_home: PathBuf,
store: PluginStore,
featured_plugin_ids_cache: RwLock<Option<CachedFeaturedPluginIds>>,
configured_marketplace_upgrade_state: RwLock<ConfiguredMarketplaceUpgradeState>,
non_curated_cache_refresh_state: RwLock<NonCuratedCacheRefreshState>,
cached_enabled_outcome: RwLock<Option<PluginLoadOutcome>>,
remote_sync_lock: Mutex<()>,
@@ -341,6 +364,9 @@ impl PluginsManager {
codex_home: codex_home.clone(),
store: PluginStore::new(codex_home),
featured_plugin_ids_cache: RwLock::new(None),
configured_marketplace_upgrade_state: RwLock::new(
ConfiguredMarketplaceUpgradeState::default(),
),
non_curated_cache_refresh_state: RwLock::new(NonCuratedCacheRefreshState::default()),
cached_enabled_outcome: RwLock::new(None),
remote_sync_lock: Mutex::new(()),
@@ -1074,6 +1100,57 @@ impl PluginsManager {
) {
if config.features.enabled(Feature::Plugins) {
self.start_curated_repo_sync();
let should_spawn_marketplace_auto_upgrade = {
let mut state = match self.configured_marketplace_upgrade_state.write() {
Ok(state) => state,
Err(err) => err.into_inner(),
};
if state.in_flight {
false
} else {
state.in_flight = true;
true
}
};
if should_spawn_marketplace_auto_upgrade {
let manager = Arc::clone(self);
let config = config.clone();
if let Err(err) = std::thread::Builder::new()
.name("plugins-marketplace-auto-upgrade".to_string())
.spawn(move || {
let outcome = manager.upgrade_configured_marketplaces_for_config(
&config, /*marketplace_name*/ None,
);
match outcome {
Ok(outcome) => {
for error in outcome.errors {
warn!(
marketplace = error.marketplace_name,
error = %error.message,
"failed to auto-upgrade configured marketplace"
);
}
}
Err(err) => {
warn!("failed to auto-upgrade configured marketplaces: {err}");
}
}
let mut state = match manager.configured_marketplace_upgrade_state.write() {
Ok(state) => state,
Err(err) => err.into_inner(),
};
state.in_flight = false;
})
{
let mut state = match self.configured_marketplace_upgrade_state.write() {
Ok(state) => state,
Err(err) => err.into_inner(),
};
state.in_flight = false;
warn!("failed to start configured marketplace auto-upgrade task: {err}");
}
}
start_startup_remote_plugin_sync_once(
Arc::clone(self),
self.codex_home.clone(),
@@ -1098,9 +1175,66 @@ impl PluginsManager {
}
}
pub fn maybe_start_non_curated_plugin_cache_refresh_for_roots(
pub fn upgrade_configured_marketplaces_for_config(
&self,
config: &Config,
marketplace_name: Option<&str>,
) -> Result<ConfiguredMarketplaceUpgradeOutcome, String> {
if let Some(marketplace_name) = marketplace_name
&& !configured_git_marketplace_names(&config.config_layer_stack)
.iter()
.any(|name| name == marketplace_name)
{
return Err(format!(
"marketplace `{marketplace_name}` is not configured as a Git marketplace"
));
}
let mut outcome = upgrade_configured_git_marketplaces(
self.codex_home.as_path(),
&config.config_layer_stack,
marketplace_name,
);
if !outcome.upgraded_roots.is_empty() {
match refresh_non_curated_plugin_cache_force_reinstall(
self.codex_home.as_path(),
&outcome.upgraded_roots,
) {
Ok(cache_refreshed) => {
if cache_refreshed {
self.clear_cache();
}
}
Err(err) => {
self.clear_cache();
outcome.errors.push(ConfiguredMarketplaceUpgradeError {
marketplace_name: marketplace_name
.unwrap_or("all configured marketplaces")
.to_string(),
message: format!(
"failed to refresh installed plugin cache after marketplace upgrade: {err}"
),
});
}
}
}
Ok(outcome)
}
pub fn maybe_start_non_curated_plugin_cache_refresh(
self: &Arc<Self>,
roots: &[AbsolutePathBuf],
) {
self.schedule_non_curated_plugin_cache_refresh(
roots,
NonCuratedCacheRefreshMode::IfVersionChanged,
);
}
fn schedule_non_curated_plugin_cache_refresh(
self: &Arc<Self>,
roots: &[AbsolutePathBuf],
mode: NonCuratedCacheRefreshMode,
) {
let mut roots = roots.to_vec();
roots.sort_unstable();
@@ -1108,6 +1242,7 @@ impl PluginsManager {
if roots.is_empty() {
return;
}
let request = NonCuratedCacheRefreshRequest { roots, mode };
let should_spawn = {
let mut state = match self.non_curated_cache_refresh_state.write() {
@@ -1115,13 +1250,25 @@ impl PluginsManager {
Err(err) => err.into_inner(),
};
// Collapse repeated plugin/list requests onto one worker and only queue another pass
// when the requested roots set actually changes.
if state.requested_roots.as_ref() == Some(&roots)
|| (!state.in_flight && state.last_refreshed_roots.as_ref() == Some(&roots))
// when the requested roots set actually changes. Forced reinstall requests are not
// deduped against the last completed pass because the same marketplace root path can
// point at newly activated files after an auto-upgrade.
if state.requested.as_ref() == Some(&request)
|| (mode == NonCuratedCacheRefreshMode::IfVersionChanged
&& !state.in_flight
&& state.last_refreshed.as_ref() == Some(&request))
{
return;
}
state.requested_roots = Some(roots);
if mode == NonCuratedCacheRefreshMode::IfVersionChanged
&& state.requested.as_ref().is_some_and(|requested| {
requested.mode == NonCuratedCacheRefreshMode::ForceReinstall
&& requested.roots == request.roots
})
{
return;
}
state.requested = Some(request);
if state.in_flight {
false
} else {
@@ -1143,7 +1290,7 @@ impl PluginsManager {
Err(err) => err.into_inner(),
};
state.in_flight = false;
state.requested_roots = None;
state.requested = None;
warn!("failed to start non-curated plugin cache refresh task: {err}");
}
}
@@ -1192,15 +1339,15 @@ impl PluginsManager {
fn run_non_curated_plugin_cache_refresh_loop(self: Arc<Self>) {
loop {
let roots = {
let request = {
let state = match self.non_curated_cache_refresh_state.read() {
Ok(state) => state,
Err(err) => err.into_inner(),
};
state.requested_roots.clone()
state.requested.clone()
};
let Some(roots) = roots else {
let Some(request) = request else {
let mut state = match self.non_curated_cache_refresh_state.write() {
Ok(state) => state,
Err(err) => err.into_inner(),
@@ -1209,30 +1356,40 @@ impl PluginsManager {
return;
};
let refreshed =
match refresh_non_curated_plugin_cache(self.codex_home.as_path(), &roots) {
Ok(cache_refreshed) => {
if cache_refreshed {
self.clear_cache();
}
true
}
Err(err) => {
let refresh_result = match request.mode {
NonCuratedCacheRefreshMode::IfVersionChanged => {
refresh_non_curated_plugin_cache(self.codex_home.as_path(), &request.roots)
}
NonCuratedCacheRefreshMode::ForceReinstall => {
refresh_non_curated_plugin_cache_force_reinstall(
self.codex_home.as_path(),
&request.roots,
)
}
};
let refreshed = match refresh_result {
Ok(cache_refreshed) => {
if cache_refreshed {
self.clear_cache();
warn!("failed to refresh non-curated plugin cache: {err}");
false
}
};
true
}
Err(err) => {
self.clear_cache();
warn!("failed to refresh non-curated plugin cache: {err}");
false
}
};
let mut state = match self.non_curated_cache_refresh_state.write() {
Ok(state) => state,
Err(err) => err.into_inner(),
};
if refreshed {
state.last_refreshed_roots = Some(roots.clone());
state.last_refreshed = Some(request.clone());
}
if state.requested_roots.as_ref() == Some(&roots) {
state.requested_roots = None;
if state.requested.as_ref() == Some(&request) {
state.requested = None;
state.in_flight = false;
return;
}
@@ -15,6 +15,8 @@ use crate::plugins::test_support::write_openai_curated_marketplace;
use codex_app_server_protocol::ConfigLayerSource;
use codex_config::McpServerConfig;
use codex_config::types::McpServerTransportConfig;
use codex_core_plugins::loader::refresh_non_curated_plugin_cache;
use codex_core_plugins::loader::refresh_non_curated_plugin_cache_force_reinstall;
use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy;
use codex_login::CodexAuth;
use codex_protocol::protocol::Product;
@@ -2676,6 +2678,68 @@ enabled = true
);
}
#[test]
fn refresh_non_curated_plugin_cache_force_reinstalls_current_local_version() {
let tmp = tempfile::tempdir().unwrap();
let repo_root = tmp.path().join("repo");
fs::create_dir_all(repo_root.join(".git")).unwrap();
fs::create_dir_all(repo_root.join(".agents/plugins")).unwrap();
write_plugin(&repo_root, "sample-plugin", "sample-plugin");
fs::write(repo_root.join("sample-plugin/skills/SKILL.md"), "new skill").unwrap();
write_file(
&repo_root.join(".agents/plugins/marketplace.json"),
r#"{
"name": "debug",
"plugins": [
{
"name": "sample-plugin",
"source": {
"source": "local",
"path": "./sample-plugin"
}
}
]
}"#,
);
write_plugin(
&tmp.path().join("plugins/cache/debug"),
"sample-plugin/local",
"sample-plugin",
);
fs::write(
tmp.path()
.join("plugins/cache/debug/sample-plugin/local/skills/SKILL.md"),
"old skill",
)
.unwrap();
write_file(
&tmp.path().join(CONFIG_TOML_FILE),
r#"[features]
plugins = true
[plugins."sample-plugin@debug"]
enabled = true
"#,
);
assert!(
refresh_non_curated_plugin_cache_force_reinstall(
tmp.path(),
&[AbsolutePathBuf::try_from(repo_root).unwrap()],
)
.expect("cache refresh should reinstall unchanged local version")
);
assert_eq!(
fs::read_to_string(
tmp.path()
.join("plugins/cache/debug/sample-plugin/local/skills/SKILL.md")
)
.unwrap(),
"new skill"
);
}
#[test]
fn refresh_non_curated_plugin_cache_ignores_invalid_unconfigured_plugin_versions() {
let tmp = tempfile::tempdir().unwrap();
@@ -38,6 +38,7 @@ pub(super) fn record_added_marketplace_entry(
let timestamp = utc_timestamp_now()?;
let update = MarketplaceConfigUpdate {
last_updated: &timestamp,
last_revision: None,
source_type: install_metadata.config_source_type(),
source: &source,
ref_name: install_metadata.ref_name(),
@@ -125,6 +125,7 @@ fn normalize_git_url(url: &str) -> String {
fn looks_like_local_path(source: &str) -> bool {
Path::new(source).is_absolute()
|| looks_like_windows_absolute_path(source)
|| source.starts_with("./")
|| source.starts_with(".\\")
|| source.starts_with("../")
@@ -134,6 +135,15 @@ fn looks_like_local_path(source: &str) -> bool {
|| source == ".."
}
fn looks_like_windows_absolute_path(source: &str) -> bool {
let bytes = source.as_bytes();
bytes.len() >= 3
&& bytes[0].is_ascii_alphabetic()
&& bytes[1] == b':'
&& matches!(bytes[2], b'\\' | b'/')
|| source.starts_with(r"\\")
}
fn resolve_local_source_path(source: &str) -> Result<PathBuf, MarketplaceAddError> {
let path = expand_tilde_path(source);
let path = if path.is_absolute() {
@@ -312,6 +322,14 @@ mod tests {
assert!(path.is_absolute());
}
#[test]
fn windows_absolute_paths_look_like_local_paths_on_every_host() {
assert!(looks_like_local_path(r"C:\Users\alice\marketplace"));
assert!(looks_like_local_path("C:/Users/alice/marketplace"));
assert!(looks_like_local_path(r"\\server\share\marketplace"));
assert!(!looks_like_local_path(r"C:relative\path"));
}
#[test]
fn local_file_source_is_rejected() {
let tempdir = TempDir::new().unwrap();
+2
View File
@@ -11,6 +11,8 @@ mod startup_sync;
#[cfg(test)]
pub(crate) mod test_support;
pub use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeError as PluginMarketplaceUpgradeError;
pub use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeOutcome as PluginMarketplaceUpgradeOutcome;
pub use codex_plugin::AppConnectorId;
pub use codex_plugin::EffectiveSkillRoots;
pub use codex_plugin::PluginCapabilitySummary;