From 24423f57124ceba1e1ae50d808caaf991c99b479 Mon Sep 17 00:00:00 2001 From: Alex Daley Date: Wed, 24 Jun 2026 17:29:11 -0400 Subject: [PATCH] [plugins] Track plugin install requests by ID (#29684) Summary - Emit `codex_plugin_install_requested` when a validated plugin install request is made, before the user accepts or declines the elicitation. - Record the exact model-visible plugin ID, remote plugin ID, required connector IDs, stable suggestion ID, and `endpoint_recommendation` vs `legacy_discovery` source. - Keep `suggest_reason` out of telemetry and leave connector-only install requests unchanged. Rollout - Backend/schema dependency: https://github.com/openai/openai/pull/1065270 - Land the backend PR before this producer starts sending the event. Validation - `just test -p codex-analytics` (83 passed) - `just test -p codex-core request_plugin_install` (17 passed) - `just fix -p codex-analytics` - `just fix -p codex-core` - `just fmt` - `git diff --check` --- .../analytics/src/analytics_client_tests.rs | 64 +++++++++++++++++++ codex-rs/analytics/src/client.rs | 15 +++++ codex-rs/analytics/src/events.rs | 51 +++++++++++++++ codex-rs/analytics/src/facts.rs | 28 ++++++++ codex-rs/analytics/src/lib.rs | 3 + codex-rs/analytics/src/reducer.rs | 20 ++++++ .../tools/handlers/request_plugin_install.rs | 39 ++++++++++- .../tests/suite/request_plugin_install.rs | 48 ++++++++++++++ 8 files changed, 266 insertions(+), 2 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 9f8df04b4..c183ad925 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -62,6 +62,10 @@ use crate::facts::HookRunInput; use crate::facts::InputError; use crate::facts::InvocationType; use crate::facts::PluginInstallFailedInput; +use crate::facts::PluginInstallRequestSource; +use crate::facts::PluginInstallRequested; +use crate::facts::PluginInstallRequestedInput; +use crate::facts::PluginInstallRequestedPlugin; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; use crate::facts::PluginUsedInput; @@ -3447,6 +3451,66 @@ async fn reducer_ingests_plugin_state_changed_fact() { ); } +#[tokio::test] +async fn reducer_ingests_plugin_install_requested_fact() { + let mut reducer = AnalyticsReducer::default(); + let mut events = Vec::new(); + let tracking = test_tracking_context("thread-1", "turn-1"); + let request = PluginInstallRequested { + suggestion_id: "request_plugin_install_call-1".to_string(), + plugins: vec![ + PluginInstallRequestedPlugin { + plugin_id: "calendar@openai-curated-remote".to_string(), + remote_plugin_id: Some("plugin_calendar".to_string()), + plugin_name: "Calendar".to_string(), + connector_ids: vec!["connector_calendar".to_string()], + }, + PluginInstallRequestedPlugin { + plugin_id: "github@openai-curated-remote".to_string(), + remote_plugin_id: None, + plugin_name: "GitHub".to_string(), + connector_ids: vec!["connector_github".to_string()], + }, + ], + source: PluginInstallRequestSource::EndpointRecommendation, + }; + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::PluginInstallRequested( + PluginInstallRequestedInput { tracking, request }, + )), + &mut events, + ) + .await; + + assert_eq!( + serde_json::to_value(&events).expect("serialize events"), + json!([{ + "event_type": "codex_plugin_install_requested", + "event_params": { + "suggestion_id": "request_plugin_install_call-1", + "plugins": [{ + "plugin_id": "calendar@openai-curated-remote", + "remote_plugin_id": "plugin_calendar", + "plugin_name": "Calendar", + "connector_ids": ["connector_calendar"], + }, { + "plugin_id": "github@openai-curated-remote", + "remote_plugin_id": null, + "plugin_name": "GitHub", + "connector_ids": ["connector_github"], + }], + "source": "endpoint_recommendation", + "thread_id": "thread-1", + "turn_id": "turn-1", + "model_slug": "gpt-5", + "product_client_id": originator().value, + } + }]) + ); +} + #[tokio::test] async fn reducer_ingests_plugin_install_failed_fact() { let mut reducer = AnalyticsReducer::default(); diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 314f0028a..5996734f8 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -16,6 +16,8 @@ use crate::facts::ExternalAgentConfigImportFailureInput; use crate::facts::HookRunFact; use crate::facts::HookRunInput; use crate::facts::PluginInstallFailedInput; +use crate::facts::PluginInstallRequested; +use crate::facts::PluginInstallRequestedInput; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; use crate::facts::SkillInvocation; @@ -310,6 +312,19 @@ impl AnalyticsEventsClient { ))); } + pub fn track_plugin_install_requested( + &self, + tracking: TrackEventsContext, + request: PluginInstallRequested, + ) { + self.record_fact(AnalyticsFact::Custom( + CustomAnalyticsFact::PluginInstallRequested(PluginInstallRequestedInput { + tracking, + request, + }), + )); + } + pub fn track_compaction(&self, event: crate::facts::CodexCompactionEvent) { self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::Compaction( Box::new(event), diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 1365e5406..ad8adf14c 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -14,6 +14,7 @@ use crate::facts::CompactionTrigger; use crate::facts::GoalEventKind; use crate::facts::HookRunFact; use crate::facts::InvocationType; +use crate::facts::PluginInstallRequested; use crate::facts::PluginState; use crate::facts::SubAgentThreadStartedInput; use crate::facts::ThreadInitializationMode; @@ -80,6 +81,7 @@ pub(crate) enum TrackEventRequest { #[allow(dead_code)] ReviewEvent(CodexReviewEventRequest), PluginUsed(CodexPluginUsedEventRequest), + PluginInstallRequested(CodexPluginInstallRequestedEventRequest), PluginInstalled(CodexPluginEventRequest), PluginUninstalled(CodexPluginEventRequest), PluginEnabled(CodexPluginEventRequest), @@ -954,6 +956,31 @@ pub(crate) struct CodexPluginUsedMetadata { pub(crate) model_slug: Option, } +#[derive(Serialize)] +pub(crate) struct CodexPluginInstallRequestedPluginMetadata { + pub(crate) plugin_id: String, + pub(crate) remote_plugin_id: Option, + pub(crate) plugin_name: String, + pub(crate) connector_ids: Vec, +} + +#[derive(Serialize)] +pub(crate) struct CodexPluginInstallRequestedMetadata { + pub(crate) suggestion_id: String, + pub(crate) plugins: Vec, + pub(crate) source: crate::facts::PluginInstallRequestSource, + pub(crate) thread_id: String, + pub(crate) turn_id: String, + pub(crate) model_slug: String, + pub(crate) product_client_id: Option, +} + +#[derive(Serialize)] +pub(crate) struct CodexPluginInstallRequestedEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: CodexPluginInstallRequestedMetadata, +} + #[derive(Serialize)] pub(crate) struct CodexPluginEventRequest { pub(crate) event_type: &'static str, @@ -1074,6 +1101,30 @@ fn codex_plugin_metadata_with_product_client_id( } } +pub(crate) fn codex_plugin_install_requested_metadata( + tracking: &TrackEventsContext, + request: PluginInstallRequested, +) -> CodexPluginInstallRequestedMetadata { + CodexPluginInstallRequestedMetadata { + suggestion_id: request.suggestion_id, + plugins: request + .plugins + .into_iter() + .map(|plugin| CodexPluginInstallRequestedPluginMetadata { + plugin_id: plugin.plugin_id, + remote_plugin_id: plugin.remote_plugin_id, + plugin_name: plugin.plugin_name, + connector_ids: plugin.connector_ids, + }) + .collect(), + source: request.source, + thread_id: tracking.thread_id.clone(), + turn_id: tracking.turn_id.clone(), + model_slug: tracking.model_slug.clone(), + product_client_id: Some(originator().value), + } +} + pub(crate) fn codex_compaction_event_params( input: CodexCompactionEvent, session_id: String, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 4505c0c69..89b870864 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -508,6 +508,7 @@ pub(crate) enum CustomAnalyticsFact { AppUsed(AppUsedInput), HookRun(HookRunInput), PluginUsed(PluginUsedInput), + PluginInstallRequested(PluginInstallRequestedInput), PluginStateChanged(PluginStateChangedInput), PluginInstallFailed(PluginInstallFailedInput), ExternalAgentConfigImportCompleted(ExternalAgentConfigImportCompletedInput), @@ -545,6 +546,33 @@ pub(crate) struct PluginUsedInput { pub plugin: PluginTelemetryMetadata, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PluginInstallRequestSource { + EndpointRecommendation, + LegacyDiscovery, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PluginInstallRequested { + pub suggestion_id: String, + pub plugins: Vec, + pub source: PluginInstallRequestSource, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PluginInstallRequestedPlugin { + pub plugin_id: String, + pub remote_plugin_id: Option, + pub plugin_name: String, + pub connector_ids: Vec, +} + +pub(crate) struct PluginInstallRequestedInput { + pub tracking: TrackEventsContext, + pub request: PluginInstallRequested, +} + pub(crate) struct PluginStateChangedInput { pub plugin: PluginTelemetryMetadata, pub state: PluginState, diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 58228f5c5..86bc9d682 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -42,6 +42,9 @@ pub use facts::GoalEventKind; pub use facts::HookRunFact; pub use facts::InputError; pub use facts::InvocationType; +pub use facts::PluginInstallRequestSource; +pub use facts::PluginInstallRequested; +pub use facts::PluginInstallRequestedPlugin; pub use facts::SkillInvocation; pub use facts::SubAgentThreadStartedInput; pub use facts::ThreadInitializationMode; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 666e756d3..522a30768 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -28,6 +28,7 @@ use crate::events::CodexOnboardingExternalAgentImportFailureMetadata; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginInstallFailedEventRequest; use crate::events::CodexPluginInstallFailedMetadata; +use crate::events::CodexPluginInstallRequestedEventRequest; use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexReviewEventParams; use crate::events::CodexReviewEventRequest; @@ -60,6 +61,7 @@ use crate::events::codex_app_metadata; use crate::events::codex_compaction_event_params; use crate::events::codex_goal_event_params; use crate::events::codex_hook_run_metadata; +use crate::events::codex_plugin_install_requested_metadata; use crate::events::codex_plugin_metadata; use crate::events::codex_plugin_used_metadata; use crate::events::plugin_state_event_type; @@ -76,6 +78,7 @@ use crate::facts::ExternalAgentConfigImportCompletedInput; use crate::facts::ExternalAgentConfigImportFailureInput; use crate::facts::HookRunInput; use crate::facts::PluginInstallFailedInput; +use crate::facts::PluginInstallRequestedInput; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; use crate::facts::PluginUsedInput; @@ -517,6 +520,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::PluginUsed(input) => { self.ingest_plugin_used(input, out); } + CustomAnalyticsFact::PluginInstallRequested(input) => { + self.ingest_plugin_install_requested(input, out); + } CustomAnalyticsFact::PluginStateChanged(input) => { self.ingest_plugin_state_changed(input, out); } @@ -775,6 +781,20 @@ impl AnalyticsReducer { })); } + fn ingest_plugin_install_requested( + &mut self, + input: PluginInstallRequestedInput, + out: &mut Vec, + ) { + let PluginInstallRequestedInput { tracking, request } = input; + out.push(TrackEventRequest::PluginInstallRequested( + CodexPluginInstallRequestedEventRequest { + event_type: "codex_plugin_install_requested", + event_params: codex_plugin_install_requested_metadata(&tracking, request), + }, + )); + } + fn ingest_plugin_state_changed( &mut self, input: PluginStateChangedInput, diff --git a/codex-rs/core/src/tools/handlers/request_plugin_install.rs b/codex-rs/core/src/tools/handlers/request_plugin_install.rs index b9656f948..ce7ce10da 100644 --- a/codex-rs/core/src/tools/handlers/request_plugin_install.rs +++ b/codex-rs/core/src/tools/handlers/request_plugin_install.rs @@ -1,6 +1,9 @@ use std::collections::HashSet; -use crate::connectors::AppInfo; +use codex_analytics::PluginInstallRequestSource; +use codex_analytics::PluginInstallRequested; +use codex_analytics::PluginInstallRequestedPlugin; +use codex_analytics::build_track_events_context; use codex_config::types::ToolSuggestDisabledTool; use codex_core_plugins::remote::REMOTE_GLOBAL_MARKETPLACE_NAME; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; @@ -29,6 +32,7 @@ use tracing::warn; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::connectors; +use crate::connectors::AppInfo; use crate::function_tool::FunctionCallError; use crate::tools::context::FunctionToolOutput; use crate::tools::context::ToolInvocation; @@ -172,7 +176,38 @@ impl RequestPluginInstallHandler { })?; let tool_type = tool.tool_type(); - let request_id = RequestId::String(format!("request_plugin_install_{call_id}").into()); + let suggestion_id = format!("request_plugin_install_{call_id}"); + if let DiscoverableTool::Plugin(plugin) = &tool { + let source = match self.presentation { + ToolSuggestPresentation::ListTool => PluginInstallRequestSource::LegacyDiscovery, + ToolSuggestPresentation::RecommendationContext => { + PluginInstallRequestSource::EndpointRecommendation + } + }; + session + .services + .analytics_events_client + .track_plugin_install_requested( + build_track_events_context( + turn.model_info.slug.clone(), + session.thread_id.to_string(), + turn.sub_id.clone(), + turn.originator.clone(), + ), + PluginInstallRequested { + suggestion_id: suggestion_id.clone(), + plugins: vec![PluginInstallRequestedPlugin { + plugin_id: plugin.id.clone(), + remote_plugin_id: plugin.remote_plugin_id.clone(), + plugin_name: plugin.name.clone(), + connector_ids: plugin.app_connector_ids.clone(), + }], + source, + }, + ); + } + + let request_id = RequestId::String(suggestion_id.into()); let request = build_request_plugin_install_elicitation_request(suggest_reason, &tool); let elicitation = session .request_mcp_server_elicitation( diff --git a/codex-rs/core/tests/suite/request_plugin_install.rs b/codex-rs/core/tests/suite/request_plugin_install.rs index f307fce05..da10cd114 100644 --- a/codex-rs/core/tests/suite/request_plugin_install.rs +++ b/codex-rs/core/tests/suite/request_plugin_install.rs @@ -38,6 +38,8 @@ use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; use serde_json::Value; use serde_json::json; +use std::time::Duration; +use std::time::Instant; use wiremock::Mock; use wiremock::MockGuard; use wiremock::ResponseTemplate; @@ -474,6 +476,52 @@ async fn run_remote_plugin_install_metadata_case() -> Result<()> { assert_eq!(meta["remote_plugin_id"], REMOTE_PLUGIN_ID); assert_eq!(meta["app_connector_ids"], json!([APP_CONNECTOR_ID])); + let deadline = Instant::now() + Duration::from_secs(10); + let analytics_event = loop { + let requests = server.received_requests().await.unwrap_or_default(); + if let Some(event) = requests + .into_iter() + .filter(|request| request.url.path() == "/codex/analytics-events/events") + .find_map(|request| { + let payload: Value = serde_json::from_slice(&request.body).ok()?; + payload["events"].as_array().and_then(|events| { + events + .iter() + .find(|event| event["event_type"] == "codex_plugin_install_requested") + .cloned() + }) + }) + { + break event; + } + if Instant::now() >= deadline { + panic!("timed out waiting for plugin install request analytics"); + } + tokio::time::sleep(Duration::from_millis(50)).await; + }; + let thread_id = analytics_event["event_params"]["thread_id"].clone(); + let turn_id = analytics_event["event_params"]["turn_id"].clone(); + assert_eq!( + analytics_event, + json!({ + "event_type": "codex_plugin_install_requested", + "event_params": { + "suggestion_id": "request_plugin_install_install-github", + "plugins": [{ + "plugin_id": "github@openai-curated-remote", + "remote_plugin_id": REMOTE_PLUGIN_ID, + "plugin_name": "GitHub", + "connector_ids": [APP_CONNECTOR_ID], + }], + "source": "endpoint_recommendation", + "thread_id": thread_id, + "turn_id": turn_id, + "model_slug": "gpt-5.4", + "product_client_id": codex_login::default_client::originator().value, + } + }) + ); + resolve_install_elicitation(&test, elicitation, ElicitationAction::Decline).await?; let requests = mock.requests();