From 58933237cda27a965927f8cc13fee56ff8e7dcef Mon Sep 17 00:00:00 2001 From: Owen Lin Date: Fri, 10 Apr 2026 17:33:58 -0700 Subject: [PATCH] feat(analytics): add guardian review event schema (#17055) Just the analytics schema definition for guardian evaluations. No wiring done yet. --- codex-rs/analytics/Cargo.toml | 2 +- codex-rs/analytics/src/client.rs | 7 + codex-rs/analytics/src/events.rs | 177 +++++++++++++++++++ codex-rs/analytics/src/facts.rs | 2 + codex-rs/analytics/src/lib.rs | 11 ++ codex-rs/analytics/src/reducer.rs | 42 +++++ codex-rs/core/src/guardian/review_session.rs | 2 +- 7 files changed, 241 insertions(+), 2 deletions(-) diff --git a/codex-rs/analytics/Cargo.toml b/codex-rs/analytics/Cargo.toml index 2239c4bc3..0f3637314 100644 --- a/codex-rs/analytics/Cargo.toml +++ b/codex-rs/analytics/Cargo.toml @@ -20,6 +20,7 @@ codex-plugin = { workspace = true } codex-protocol = { workspace = true } os_info = { workspace = true } serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } sha1 = { workspace = true } tokio = { workspace = true, features = [ "macros", @@ -29,4 +30,3 @@ tracing = { workspace = true, features = ["log"] } [dev-dependencies] pretty_assertions = { workspace = true } -serde_json = { workspace = true } diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index ef9c42cb1..0d96ee606 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -1,4 +1,5 @@ use crate::events::AppServerRpcTransport; +use crate::events::GuardianReviewEventParams; use crate::events::TrackEventRequest; use crate::events::TrackEventsRequest; use crate::events::current_runtime_metadata; @@ -151,6 +152,12 @@ impl AnalyticsEventsClient { )); } + pub fn track_guardian_review(&self, input: GuardianReviewEventParams) { + self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::GuardianReview( + Box::new(input), + ))); + } + pub fn track_app_mentioned(&self, tracking: TrackEventsContext, mentions: Vec) { if mentions.is_empty() { return; diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 97ebe1f04..618dd8ffe 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -6,6 +6,9 @@ use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; use codex_login::default_client::originator; use codex_plugin::PluginTelemetryMetadata; +use codex_protocol::approvals::NetworkApprovalProtocol; +use codex_protocol::models::PermissionProfile; +use codex_protocol::models::SandboxPermissions; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; use serde::Serialize; @@ -36,6 +39,7 @@ pub(crate) struct TrackEventsRequest { pub(crate) enum TrackEventRequest { SkillInvocation(SkillInvocationEventRequest), ThreadInitialized(ThreadInitializedEvent), + GuardianReview(Box), AppMentioned(CodexAppMentionedEventRequest), AppUsed(CodexAppUsedEventRequest), Compaction(Box), @@ -101,6 +105,179 @@ pub(crate) struct ThreadInitializedEvent { pub(crate) event_params: ThreadInitializedEventParams, } +#[derive(Serialize)] +pub(crate) struct GuardianReviewEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: GuardianReviewEventPayload, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GuardianReviewDecision { + Approved, + Denied, + Aborted, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GuardianReviewTerminalStatus { + Approved, + Denied, + Aborted, + TimedOut, + FailedClosed, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GuardianReviewFailureReason { + Timeout, + Cancelled, + PromptBuildError, + SessionError, + ParseError, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GuardianReviewSessionKind { + TrunkNew, + TrunkReused, + EphemeralForked, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum GuardianReviewRiskLevel { + Low, + Medium, + High, + Critical, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum GuardianReviewUserAuthorization { + Unknown, + Low, + Medium, + High, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum GuardianReviewOutcome { + Allow, + Deny, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GuardianApprovalRequestSource { + /// Approval requested directly by the main Codex turn. + MainTurn, + /// Approval requested by a delegated subagent and routed through the parent + /// session for guardian review. + DelegatedSubagent, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum GuardianReviewedAction { + Shell { + command: Vec, + command_display: String, + cwd: String, + sandbox_permissions: SandboxPermissions, + additional_permissions: Option, + justification: Option, + }, + UnifiedExec { + command: Vec, + command_display: String, + cwd: String, + sandbox_permissions: SandboxPermissions, + additional_permissions: Option, + justification: Option, + tty: bool, + }, + Execve { + source: GuardianCommandSource, + program: String, + argv: Vec, + cwd: String, + additional_permissions: Option, + }, + ApplyPatch { + cwd: String, + files: Vec, + }, + NetworkAccess { + target: String, + host: String, + protocol: NetworkApprovalProtocol, + port: u16, + }, + McpToolCall { + server: String, + tool_name: String, + connector_id: Option, + connector_name: Option, + tool_title: Option, + }, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GuardianCommandSource { + Shell, + UnifiedExec, +} + +#[derive(Clone, Serialize)] +pub struct GuardianReviewEventParams { + pub thread_id: String, + pub turn_id: String, + pub review_id: String, + pub target_item_id: String, + pub retry_reason: Option, + pub approval_request_source: GuardianApprovalRequestSource, + pub reviewed_action: GuardianReviewedAction, + pub reviewed_action_truncated: bool, + pub decision: GuardianReviewDecision, + pub terminal_status: GuardianReviewTerminalStatus, + pub failure_reason: Option, + pub risk_level: Option, + pub user_authorization: Option, + pub outcome: Option, + pub rationale: Option, + pub guardian_thread_id: Option, + pub guardian_session_kind: Option, + pub guardian_model: Option, + pub guardian_reasoning_effort: Option, + pub had_prior_review_context: Option, + pub review_timeout_ms: u64, + pub tool_call_count: u64, + pub time_to_first_token_ms: Option, + pub completion_latency_ms: Option, + pub started_at: u64, + pub completed_at: Option, + pub input_tokens: Option, + pub cached_input_tokens: Option, + pub output_tokens: Option, + pub reasoning_output_tokens: Option, + pub total_tokens: Option, +} + +#[derive(Serialize)] +pub(crate) struct GuardianReviewEventPayload { + pub(crate) app_server_client: CodexAppServerClientMetadata, + pub(crate) runtime: CodexRuntimeMetadata, + #[serde(flatten)] + pub(crate) guardian_review: GuardianReviewEventParams, +} + #[derive(Serialize)] pub(crate) struct CodexAppMetadata { pub(crate) connector_id: Option, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 16c482abd..931ae0101 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -1,5 +1,6 @@ use crate::events::AppServerRpcTransport; use crate::events::CodexRuntimeMetadata; +use crate::events::GuardianReviewEventParams; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::InitializeParams; @@ -154,6 +155,7 @@ pub(crate) enum AnalyticsFact { pub(crate) enum CustomAnalyticsFact { SubAgentThreadStarted(SubAgentThreadStartedInput), Compaction(Box), + GuardianReview(Box), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), AppUsed(AppUsedInput), diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 0e7ef5925..9b4cc1e9b 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -5,6 +5,17 @@ mod reducer; pub use client::AnalyticsEventsClient; pub use events::AppServerRpcTransport; +pub use events::GuardianApprovalRequestSource; +pub use events::GuardianCommandSource; +pub use events::GuardianReviewDecision; +pub use events::GuardianReviewEventParams; +pub use events::GuardianReviewFailureReason; +pub use events::GuardianReviewOutcome; +pub use events::GuardianReviewRiskLevel; +pub use events::GuardianReviewSessionKind; +pub use events::GuardianReviewTerminalStatus; +pub use events::GuardianReviewUserAuthorization; +pub use events::GuardianReviewedAction; pub use facts::AppInvocation; pub use facts::CodexCompactionEvent; pub use facts::CompactionImplementation; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 7665b7bfd..0ed289976 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -6,6 +6,9 @@ use crate::events::CodexCompactionEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; +use crate::events::GuardianReviewEventParams; +use crate::events::GuardianReviewEventPayload; +use crate::events::GuardianReviewEventRequest; use crate::events::SkillInvocationEventParams; use crate::events::SkillInvocationEventRequest; use crate::events::ThreadInitializationMode; @@ -120,6 +123,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::Compaction(input) => { self.ingest_compaction(*input, out); } + CustomAnalyticsFact::GuardianReview(input) => { + self.ingest_guardian_review(*input, out); + } CustomAnalyticsFact::SkillInvoked(input) => { self.ingest_skill_invoked(input, out).await; } @@ -174,6 +180,42 @@ impl AnalyticsReducer { )); } + fn ingest_guardian_review( + &mut self, + input: GuardianReviewEventParams, + out: &mut Vec, + ) { + let Some(connection_id) = self.thread_connections.get(&input.thread_id) else { + tracing::warn!( + thread_id = %input.thread_id, + turn_id = %input.turn_id, + review_id = %input.review_id, + "dropping guardian analytics event: missing thread connection metadata" + ); + return; + }; + let Some(connection_state) = self.connections.get(connection_id) else { + tracing::warn!( + thread_id = %input.thread_id, + turn_id = %input.turn_id, + review_id = %input.review_id, + connection_id, + "dropping guardian analytics event: missing connection metadata" + ); + return; + }; + out.push(TrackEventRequest::GuardianReview(Box::new( + GuardianReviewEventRequest { + event_type: "codex_guardian_review", + event_params: GuardianReviewEventPayload { + app_server_client: connection_state.app_server_client.clone(), + runtime: connection_state.runtime.clone(), + guardian_review: input, + }, + }, + ))); + } + async fn ingest_skill_invoked( &mut self, input: SkillInvokedInput, diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs index 256977f51..5fe194cac 100644 --- a/codex-rs/core/src/guardian/review_session.rs +++ b/codex-rs/core/src/guardian/review_session.rs @@ -414,7 +414,7 @@ impl GuardianReviewSessionManager { let snapshot = state.last_committed_fork_snapshot.as_ref()?; match &snapshot.initial_history { InitialHistory::Forked(items) => Some(items.clone()), - InitialHistory::New | InitialHistory::Resumed(_) => None, + _ => None, } }