[codex] Track plugin install and import telemetry failures (#28731)

## Summary
- Track plugin install failures through the unified
`codex_plugin_install_failed` event for local installs, remote install
preflight failures, bundle failures, and remote catalog/backend
failures.
- Send classified `error_type` values in plugin install failure
analytics instead of raw error strings.
- Stop sending raw external-agent import errors in analytics while
preserving raw failure details in app-facing import
notifications/history.
- Keep raw plugin/migration diagnostics in `tracing::warn!` logs.
- Keep remote failure plugin names as the existing local placeholder
(`unknown`) and remove the extra telemetry plugin-name override.
- Change `ExternalAgentConfigImportParams.source` from a generated enum
to `string | null`, with legacy `claudeCode` / `claudeCowork` inputs
normalized to existing analytics values.

## Testing
This commit is contained in:
charlesgong-openai
2026-06-17 13:16:34 -07:00
committed by GitHub
parent 5867b529ae
commit 3959ab0ffc
25 changed files with 1314 additions and 97 deletions
@@ -237,7 +237,7 @@ impl ExternalAgentConfigService {
pub(crate) async fn import(
&self,
migration_items: Vec<ExternalAgentConfigMigrationItem>,
) -> io::Result<ExternalAgentConfigImportOutcome> {
) -> ExternalAgentConfigImportOutcome {
let mut outcome = ExternalAgentConfigImportOutcome::default();
for migration_item in migration_items {
let item_type = migration_item.item_type;
@@ -413,17 +413,28 @@ impl ExternalAgentConfigService {
})(),
ExternalAgentConfigMigrationItemType::Sessions => Ok(()),
};
if let Err(err) = import_result {
if item_type == ExternalAgentConfigMigrationItemType::Plugins {
outcome.item_results.push(item_result);
continue;
}
return Err(err);
if let Err(err) = import_result
&& item_type != ExternalAgentConfigMigrationItemType::Plugins
{
let message = err.to_string();
let error_type = if message.contains("invalid existing config.toml") {
"invalid_existing_config"
} else {
"external_agent_config_import_error"
};
item_result.record_error(ExternalAgentConfigImportRawError {
item_type,
error_type: Some(error_type.to_string()),
failure_stage: "import_request_failed".to_string(),
message,
cwd: item_result.cwd.clone(),
source: None,
});
}
outcome.item_results.push(item_result);
}
Ok(outcome)
outcome
}
async fn detect_migrations(
@@ -561,8 +561,7 @@ async fn import_repo_migrates_mcp_hooks_commands_and_subagents() {
details: None,
},
])
.await
.expect("import");
.await;
let config: TomlValue = toml::from_str(
&fs::read_to_string(repo_root.join(".codex").join("config.toml")).expect("read config"),
@@ -724,8 +723,7 @@ url = "https://example.com/mixed-transport"
cwd: Some(repo_root.clone()),
details: None,
}])
.await
.expect("import");
.await;
assert_eq!(
fs::read_to_string(repo_root.join(".codex").join("config.toml")).expect("read config"),
@@ -838,8 +836,7 @@ async fn import_home_migrates_supported_config_fields_skills_and_agents_md() {
details: None,
},
])
.await
.expect("import");
.await;
assert_eq!(
fs::read_to_string(codex_home.join("AGENTS.md")).expect("read agents"),
@@ -894,8 +891,7 @@ async fn import_home_config_uses_local_settings_over_project_settings() {
cwd: None,
details: None,
}])
.await
.expect("import");
.await;
let config: TomlValue =
toml::from_str(&fs::read_to_string(codex_home.join("config.toml")).expect("read config"))
@@ -939,8 +935,7 @@ async fn import_home_config_ignores_invalid_local_settings() {
cwd: None,
details: None,
}])
.await
.expect("import");
.await;
assert_eq!(
fs::read_to_string(codex_home.join("config.toml")).expect("read config"),
@@ -965,8 +960,7 @@ async fn import_home_skips_empty_config_migration() {
cwd: None,
details: None,
}])
.await
.expect("import");
.await;
assert_eq!(
outcome.item_results,
@@ -1043,8 +1037,7 @@ async fn import_local_plugins_returns_completed_status() {
..Default::default()
}),
}])
.await
.expect("import");
.await;
assert_eq!(
outcome.pending_plugin_imports,
@@ -1104,8 +1097,7 @@ async fn import_git_plugins_returns_pending_async_status() {
..Default::default()
}),
}])
.await
.expect("import");
.await;
assert_eq!(
outcome.pending_plugin_imports,
@@ -1235,8 +1227,7 @@ async fn import_repo_agents_md_rewrites_terms_and_skips_non_empty_targets() {
details: None,
},
])
.await
.expect("import");
.await;
assert_eq!(
outcome.item_results,
@@ -1302,8 +1293,7 @@ async fn import_repo_agents_md_overwrites_empty_targets() {
cwd: Some(repo_root.clone()),
details: None,
}])
.await
.expect("import");
.await;
assert_eq!(
outcome.item_results,
@@ -1403,8 +1393,7 @@ async fn import_repo_hooks_preserves_disabled_codex_hooks_feature() {
cwd: Some(repo_root.clone()),
details: None,
}])
.await
.expect("import");
.await;
assert_eq!(
outcome.item_results,
@@ -1481,8 +1470,7 @@ async fn import_repo_mcp_uses_home_settings_toggles_when_repo_settings_missing()
cwd: Some(repo_root.clone()),
details: None,
}])
.await
.expect("import");
.await;
assert_eq!(
outcome.item_results,
@@ -1559,8 +1547,7 @@ async fn import_repo_mcp_uses_local_settings_toggles_over_project_settings() {
cwd: Some(repo_root.clone()),
details: None,
}])
.await
.expect("import");
.await;
let config: TomlValue = toml::from_str(
&fs::read_to_string(repo_root.join(".codex").join("config.toml")).expect("read config"),
@@ -1607,8 +1594,7 @@ async fn import_repo_mcp_ignores_invalid_home_settings_when_repo_settings_missin
cwd: Some(repo_root.clone()),
details: None,
}])
.await
.expect("import");
.await;
let config: TomlValue = toml::from_str(
&fs::read_to_string(repo_root.join(".codex").join("config.toml")).expect("read config"),
@@ -1649,8 +1635,7 @@ async fn import_repo_uses_non_empty_external_agent_agents_source() {
cwd: Some(repo_root.clone()),
details: None,
}])
.await
.expect("import");
.await;
assert_eq!(
fs::read_to_string(repo_root.join("AGENTS.md")).expect("read target"),
@@ -1683,8 +1668,7 @@ async fn import_continues_after_failed_migration_item() {
details: None,
},
])
.await
.expect("import continues");
.await;
assert_eq!(
fs::read_to_string(repo_root.join("AGENTS.md")).expect("read target"),
+2 -1
View File
@@ -515,7 +515,7 @@ impl MessageProcessor {
outgoing.clone(),
config_manager.clone(),
thread_manager.clone(),
analytics_events_client,
analytics_events_client.clone(),
);
let external_agent_config_processor =
ExternalAgentConfigRequestProcessor::new(ExternalAgentConfigRequestProcessorArgs {
@@ -525,6 +525,7 @@ impl MessageProcessor {
config_manager: config_manager.clone(),
config_processor: config_processor.clone(),
state_db,
analytics_events_client,
arg0_paths,
codex_home: config.codex_home.to_path_buf(),
});
@@ -15,6 +15,9 @@ use crate::config_manager::ConfigManager;
use crate::error_code::internal_error;
use crate::outgoing_message::ConnectionRequestId;
use crate::outgoing_message::OutgoingMessageSender;
use codex_analytics::AnalyticsEventsClient;
use codex_analytics::ExternalAgentConfigImportCompletedInput;
use codex_analytics::ExternalAgentConfigImportFailureInput;
use codex_app_server_protocol::CommandMigration;
use codex_app_server_protocol::ExternalAgentConfigDetectParams;
use codex_app_server_protocol::ExternalAgentConfigDetectResponse;
@@ -57,6 +60,7 @@ pub(crate) struct ExternalAgentConfigRequestProcessor {
thread_manager: Arc<ThreadManager>,
config_processor: ConfigRequestProcessor,
state_db: Option<StateDbHandle>,
analytics_events_client: AnalyticsEventsClient,
}
pub(crate) struct ExternalAgentConfigRequestProcessorArgs {
@@ -66,6 +70,7 @@ pub(crate) struct ExternalAgentConfigRequestProcessorArgs {
pub(crate) config_manager: ConfigManager,
pub(crate) config_processor: ConfigRequestProcessor,
pub(crate) state_db: Option<StateDbHandle>,
pub(crate) analytics_events_client: AnalyticsEventsClient,
pub(crate) arg0_paths: Arg0DispatchPaths,
pub(crate) codex_home: PathBuf,
}
@@ -79,6 +84,7 @@ impl ExternalAgentConfigRequestProcessor {
config_manager,
config_processor,
state_db,
analytics_events_client,
arg0_paths,
codex_home,
} = args;
@@ -96,6 +102,7 @@ impl ExternalAgentConfigRequestProcessor {
thread_manager,
config_processor,
state_db,
analytics_events_client,
}
}
@@ -199,6 +206,7 @@ impl ExternalAgentConfigRequestProcessor {
params: ExternalAgentConfigImportParams,
) -> Result<(), JSONRPCErrorError> {
let import_id = Uuid::new_v4().to_string();
let analytics_source = params.source.clone().unwrap_or_default();
let needs_runtime_refresh = migration_items_need_runtime_refresh(&params.migration_items);
let has_migration_items = !params.migration_items.is_empty();
let has_plugin_imports = params.migration_items.iter().any(|item| {
@@ -209,7 +217,7 @@ impl ExternalAgentConfigRequestProcessor {
});
let (pending_session_imports, session_validation_result) =
self.validate_pending_session_imports(&params);
let import_outcome = self.import_external_agent_config(params).await?;
let import_outcome = self.import_external_agent_config(params).await;
if needs_runtime_refresh {
self.config_processor.handle_config_mutation().await;
}
@@ -242,7 +250,9 @@ impl ExternalAgentConfigRequestProcessor {
send_completed_import_notification(
&self.outgoing,
self.state_db.as_ref(),
&self.analytics_events_client,
import_id,
analytics_source,
&completed_item_results,
)
.await;
@@ -253,6 +263,7 @@ impl ExternalAgentConfigRequestProcessor {
let plugin_processor = self.clone();
let outgoing = Arc::clone(&self.outgoing);
let state_db = self.state_db.clone();
let analytics_events_client = self.analytics_events_client.clone();
let thread_manager = Arc::clone(&self.thread_manager);
let session_import_result = (!pending_session_imports.is_empty()).then(|| {
CoreImportItemResult::new(
@@ -324,7 +335,9 @@ impl ExternalAgentConfigRequestProcessor {
send_completed_import_notification(
&outgoing,
state_db.as_ref(),
&analytics_events_client,
import_id,
analytics_source,
&completed_item_results,
)
.await;
@@ -421,7 +434,7 @@ impl ExternalAgentConfigRequestProcessor {
async fn import_external_agent_config(
&self,
params: ExternalAgentConfigImportParams,
) -> Result<CoreImportOutcome, JSONRPCErrorError> {
) -> CoreImportOutcome {
self.migration_service
.import(
params
@@ -516,7 +529,6 @@ impl ExternalAgentConfigRequestProcessor {
.collect(),
)
.await
.map_err(|err| internal_error(err.to_string()))
}
async fn complete_pending_plugin_import(
@@ -551,10 +563,14 @@ async fn send_import_progress(
async fn send_completed_import_notification(
outgoing: &OutgoingMessageSender,
state_db: Option<&StateDbHandle>,
analytics_events_client: &AnalyticsEventsClient,
import_id: String,
analytics_source: String,
item_results: &[CoreImportItemResult],
) {
let notification = completed_notification(import_id, item_results);
log_completed_import_failures(&notification);
track_completed_import_notification(analytics_events_client, &analytics_source, &notification);
if let Some(state_db) = state_db
&& let Err(err) = record_completed_import_notification(state_db, &notification).await
{
@@ -571,6 +587,75 @@ async fn send_completed_import_notification(
.await;
}
fn log_completed_import_failures(notification: &ExternalAgentConfigImportCompletedNotification) {
for type_result in &notification.item_type_results {
for failure in &type_result.failures {
let error_type = import_failure_error_type(failure);
tracing::warn!(
import_id = %notification.import_id,
item_type = ?failure.item_type,
error_type = %error_type,
failure_stage = %failure.failure_stage,
cwd = ?failure.cwd,
source = ?failure.source,
error = %failure.message,
"external agent config migration item failed"
);
}
}
}
fn track_completed_import_notification(
analytics_events_client: &AnalyticsEventsClient,
analytics_source: &str,
notification: &ExternalAgentConfigImportCompletedNotification,
) {
for type_result in &notification.item_type_results {
let item_type = analytics_migration_item_type(type_result.item_type).to_string();
analytics_events_client.track_external_agent_config_import_completed(
ExternalAgentConfigImportCompletedInput {
import_id: notification.import_id.clone(),
source: analytics_source.to_string(),
item_type: item_type.clone(),
success_count: type_result.successes.len(),
failed_count: type_result.failures.len(),
},
);
for failure in &type_result.failures {
analytics_events_client.track_external_agent_config_import_failure(
ExternalAgentConfigImportFailureInput {
import_id: notification.import_id.clone(),
source: analytics_source.to_string(),
item_type: item_type.clone(),
failure_stage: failure.failure_stage.clone(),
error_type: import_failure_error_type(failure),
},
);
}
}
}
fn import_failure_error_type(failure: &ProtocolImportFailure) -> String {
failure
.error_type
.clone()
.unwrap_or_else(|| failure.failure_stage.clone())
}
fn analytics_migration_item_type(item_type: ExternalAgentConfigMigrationItemType) -> &'static str {
match item_type {
ExternalAgentConfigMigrationItemType::AgentsMd => "AGENTS_MD",
ExternalAgentConfigMigrationItemType::Config => "CONFIG",
ExternalAgentConfigMigrationItemType::Skills => "SKILLS",
ExternalAgentConfigMigrationItemType::Plugins => "PLUGINS",
ExternalAgentConfigMigrationItemType::McpServerConfig => "MCP_SERVER_CONFIG",
ExternalAgentConfigMigrationItemType::Subagents => "SUBAGENTS",
ExternalAgentConfigMigrationItemType::Hooks => "HOOKS",
ExternalAgentConfigMigrationItemType::Commands => "COMMANDS",
ExternalAgentConfigMigrationItemType::Sessions => "SESSIONS",
}
}
async fn record_completed_import_notification(
state_db: &StateDbHandle,
notification: &ExternalAgentConfigImportCompletedNotification,
@@ -18,9 +18,12 @@ use codex_core_plugins::remote::REMOTE_WORKSPACE_SHARED_WITH_ME_UNLISTED_MARKETP
use codex_core_plugins::remote::RemoteAppTemplateUnavailableReason;
use codex_core_plugins::remote::is_valid_remote_plugin_id;
use codex_core_plugins::remote::validate_remote_plugin_id;
use codex_core_plugins::remote_bundle::RemotePluginBundleInstallError;
use codex_mcp::McpOAuthLoginSupport;
use codex_mcp::oauth_login_support;
use codex_mcp::should_retry_without_scopes;
use codex_plugin::PluginId;
use codex_plugin::PluginTelemetryMetadata;
use codex_rmcp_client::perform_oauth_login_silent;
#[derive(Clone)]
@@ -1437,15 +1440,24 @@ impl PluginRequestProcessor {
}
let plugins_manager = self.thread_manager.plugins_manager();
let marketplace_display = marketplace_path.display().to_string();
let plugin_name_for_log = plugin_name.clone();
let request = PluginInstallRequest {
plugin_name,
marketplace_path,
};
let result = plugins_manager
.install_plugin(request)
.await
.map_err(Self::plugin_install_error)?;
let result = match plugins_manager.install_plugin(request).await {
Ok(result) => result,
Err(err) => {
warn!(
marketplace = %marketplace_display,
plugin_name = %plugin_name_for_log,
"failed to install plugin: {err}"
);
return Err(Self::plugin_install_error(err));
}
};
let config = match self.load_latest_config(config_cwd).await {
Ok(config) => config,
Err(err) => {
@@ -1512,11 +1524,20 @@ impl PluginRequestProcessor {
)
.await
.map_err(|err| {
let error_type = remote_plugin_catalog_error_type(&err);
self.track_plugin_install_failed_for_remote_plugin(
&remote_plugin_id,
&remote_marketplace_name,
error_type,
err.to_string(),
);
remote_plugin_catalog_error_to_jsonrpc(
err,
"read remote plugin details before install",
)
})?;
let actual_remote_marketplace_name = remote_detail.marketplace_name.clone();
let remote_plugin_name = remote_detail.summary.name.clone();
if remote_detail.summary.availability == PluginAvailability::DisabledByAdmin {
return Err(invalid_request(format!(
"remote plugin {remote_plugin_id} is disabled by admin"
@@ -1527,31 +1548,48 @@ impl PluginRequestProcessor {
"remote plugin {remote_plugin_id} is not available for install"
)));
}
let actual_remote_marketplace_name = remote_detail.marketplace_name.clone();
// Direct install writes the same cache tree that installed-plugin sync
// prunes before the backend installed snapshot can include this plugin.
let _remote_plugin_cache_mutation =
codex_core_plugins::remote::mark_remote_plugin_cache_mutation_in_flight(
config.codex_home.as_path(),
&actual_remote_marketplace_name,
&remote_detail.summary.name,
&remote_plugin_name,
);
let validated_bundle = codex_core_plugins::remote_bundle::validate_remote_plugin_bundle(
&remote_plugin_id,
&actual_remote_marketplace_name,
&remote_detail.summary.name,
&remote_plugin_name,
remote_detail.release_version.as_deref(),
remote_detail.bundle_download_url.as_deref(),
remote_detail.app_manifest.clone(),
)
.map_err(remote_plugin_bundle_install_error_to_jsonrpc)?;
.map_err(|err| {
let error_type = remote_plugin_bundle_install_error_type(&err);
self.track_plugin_install_failed_for_remote_plugin(
&remote_plugin_id,
&actual_remote_marketplace_name,
error_type,
err.to_string(),
);
remote_plugin_bundle_install_error_to_jsonrpc(err)
})?;
let result = codex_core_plugins::remote_bundle::download_and_install_remote_plugin_bundle(
config.codex_home.to_path_buf(),
validated_bundle,
)
.await
.map_err(remote_plugin_bundle_install_error_to_jsonrpc)?;
.map_err(|err| {
let error_type = remote_plugin_bundle_install_error_type(&err);
self.track_plugin_install_failed_for_remote_plugin(
&remote_plugin_id,
&actual_remote_marketplace_name,
error_type,
err.to_string(),
);
remote_plugin_bundle_install_error_to_jsonrpc(err)
})?;
// Cache first so a backend install cannot succeed when local materialization fails.
// If this backend call fails, the cache entry is harmless because remote installed state
@@ -1563,7 +1601,16 @@ impl PluginRequestProcessor {
&remote_plugin_id,
)
.await
.map_err(|err| remote_plugin_catalog_error_to_jsonrpc(err, "install remote plugin"))?;
.map_err(|err| {
let error_type = remote_plugin_catalog_error_type(&err);
self.track_plugin_install_failed_for_remote_plugin(
&remote_plugin_id,
&actual_remote_marketplace_name,
error_type,
err.to_string(),
);
remote_plugin_catalog_error_to_jsonrpc(err, "install remote plugin")
})?;
self.thread_manager
.plugins_manager()
@@ -1646,6 +1693,32 @@ impl PluginRequestProcessor {
})
}
fn track_plugin_install_failed_for_remote_plugin(
&self,
remote_plugin_id: &str,
marketplace_name: &str,
error_type: &'static str,
error_message: String,
) {
tracing::warn!(
remote_plugin_id = %remote_plugin_id,
marketplace_name = %marketplace_name,
error_type = %error_type,
error = %error_message,
"remote plugin install failed"
);
// The remote id is reported separately; this local name only satisfies
// PluginId validation before remote details are available.
let Ok(plugin_id) = PluginId::new("unknown".to_string(), marketplace_name.to_string())
else {
return;
};
let mut plugin = PluginTelemetryMetadata::from_plugin_id(&plugin_id);
plugin.remote_plugin_id = Some(remote_plugin_id.to_string());
self.analytics_events_client
.track_plugin_install_failed(plugin, error_type.to_string());
}
async fn plugin_apps_needing_auth_for_install(
&self,
config: &Config,
@@ -2145,6 +2218,75 @@ fn remote_plugin_detail_to_info(
}
}
fn remote_plugin_catalog_error_type(err: &RemotePluginCatalogError) -> &'static str {
match err {
RemotePluginCatalogError::AuthRequired => "remote_catalog_auth_required",
RemotePluginCatalogError::UnsupportedAuthMode => "remote_catalog_unsupported_auth_mode",
RemotePluginCatalogError::AuthToken(_) => "remote_catalog_auth_token",
RemotePluginCatalogError::Request { .. } => "remote_catalog_request",
RemotePluginCatalogError::UnexpectedStatus { .. } => "remote_catalog_unexpected_status",
RemotePluginCatalogError::Decode { .. } => "remote_catalog_decode",
RemotePluginCatalogError::InvalidBaseUrl(_) => "remote_catalog_invalid_base_url",
RemotePluginCatalogError::InvalidBaseUrlPath => "remote_catalog_invalid_base_url_path",
RemotePluginCatalogError::UnknownMarketplace { .. } => "remote_catalog_unknown_marketplace",
RemotePluginCatalogError::UnexpectedPluginId { .. } => {
"remote_catalog_unexpected_plugin_id"
}
RemotePluginCatalogError::UnexpectedSkillName { .. } => {
"remote_catalog_unexpected_skill_name"
}
RemotePluginCatalogError::UnexpectedEnabledState { .. } => {
"remote_catalog_unexpected_enabled_state"
}
RemotePluginCatalogError::InvalidPluginPath { .. } => "remote_catalog_invalid_plugin_path",
RemotePluginCatalogError::PluginShareCheckoutNotAvailable { .. } => {
"remote_catalog_plugin_share_checkout_not_available"
}
RemotePluginCatalogError::Archive { .. } => "remote_catalog_archive",
RemotePluginCatalogError::ArchiveJoin(_) => "remote_catalog_archive_join",
RemotePluginCatalogError::ArchiveTooLarge { .. } => "remote_catalog_archive_too_large",
RemotePluginCatalogError::MissingUploadEtag => "remote_catalog_missing_upload_etag",
RemotePluginCatalogError::UnexpectedResponse(_) => "remote_catalog_unexpected_response",
RemotePluginCatalogError::CacheRemove(_) => "remote_catalog_cache_remove",
}
}
fn remote_plugin_bundle_install_error_type(err: &RemotePluginBundleInstallError) -> &'static str {
match err {
RemotePluginBundleInstallError::MissingReleaseVersion { .. } => {
"remote_bundle_missing_release_version"
}
RemotePluginBundleInstallError::InvalidReleaseVersion { .. } => {
"remote_bundle_invalid_release_version"
}
RemotePluginBundleInstallError::MissingBundleDownloadUrl { .. } => {
"remote_bundle_missing_download_url"
}
RemotePluginBundleInstallError::InvalidBundleDownloadUrl { .. } => {
"remote_bundle_invalid_download_url"
}
RemotePluginBundleInstallError::UnsupportedBundleDownloadUrlScheme { .. } => {
"remote_bundle_unsupported_download_url_scheme"
}
RemotePluginBundleInstallError::InvalidPluginId { .. } => "remote_bundle_invalid_plugin_id",
RemotePluginBundleInstallError::DownloadRequest { .. } => "remote_bundle_download_request",
RemotePluginBundleInstallError::DownloadStatus { .. } => "remote_bundle_download_status",
RemotePluginBundleInstallError::DownloadBody { .. } => "remote_bundle_download_body",
RemotePluginBundleInstallError::DownloadTooLarge { .. } => {
"remote_bundle_download_too_large"
}
RemotePluginBundleInstallError::UnsupportedBundleDownloadFinalUrl { .. } => {
"remote_bundle_unsupported_download_final_url"
}
RemotePluginBundleInstallError::ExtractedBundleTooLarge { .. } => {
"remote_bundle_extracted_too_large"
}
RemotePluginBundleInstallError::Io { .. } => "remote_bundle_io",
RemotePluginBundleInstallError::InvalidBundle(_) => "remote_bundle_invalid_bundle",
RemotePluginBundleInstallError::Store(_) => "remote_bundle_store",
}
}
fn remote_plugin_catalog_error_to_jsonrpc(
err: RemotePluginCatalogError,
context: &str,