Refactor execpolicy fallback evaluation (#7544)

## Refactor of the `execpolicy` crate

To illustrate why we need this refactor, consider an agent attempting to
run `apple | rm -rf ./`. Suppose `apple` is allowed by `execpolicy`.
Before this PR, `execpolicy` would consider `apple` and `pear` and only
render one rule match: `Allow`. We would skip any heuristics checks on
`rm -rf ./` and immediately approve `apple | rm -rf ./` to run.

To fix this, we now thread a `fallback` evaluation function into
`execpolicy` that runs when no `execpolicy` rules match a given command.
In our example, we would run `fallback` on `rm -rf ./` and prevent
`apple | rm -rf ./` from being run without approval.

codex-rs/execpolicy/src/execpolicycheck.rs

@@ -4,10 +4,12 @@ use std::path::PathBuf;
 use anyhow::Context;
 use anyhow::Result;
 use clap::Parser;
+use serde::Serialize;
 
-use crate::Evaluation;
+use crate::Decision;
 use crate::Policy;
 use crate::PolicyParser;
+use crate::RuleMatch;
 
 /// Arguments for evaluating a command against one or more execpolicy files.
 #[derive(Debug, Parser, Clone)]
@@ -34,20 +36,25 @@ impl ExecPolicyCheckCommand {
     /// Load the policies for this command, evaluate the command, and render JSON output.
     pub fn run(&self) -> Result<()> {
         let policy = load_policies(&self.policies)?;
-        let evaluation = policy.check(&self.command);
+        let matched_rules = policy.matches_for_command(&self.command, None);
 
-        let json = format_evaluation_json(&evaluation, self.pretty)?;
+        let json = format_matches_json(&matched_rules, self.pretty)?;
         println!("{json}");
 
         Ok(())
     }
 }
 
-pub fn format_evaluation_json(evaluation: &Evaluation, pretty: bool) -> Result<String> {
+pub fn format_matches_json(matched_rules: &[RuleMatch], pretty: bool) -> Result<String> {
+    let output = ExecPolicyCheckOutput {
+        matched_rules,
+        decision: matched_rules.iter().map(RuleMatch::decision).max(),
+    };
+
     if pretty {
-        serde_json::to_string_pretty(evaluation).map_err(Into::into)
+        serde_json::to_string_pretty(&output).map_err(Into::into)
     } else {
-        serde_json::to_string(evaluation).map_err(Into::into)
+        serde_json::to_string(&output).map_err(Into::into)
     }
 }
 
@@ -65,3 +72,12 @@ pub fn load_policies(policy_paths: &[PathBuf]) -> Result<Policy> {
 
     Ok(parser.build())
 }
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+struct ExecPolicyCheckOutput<'a> {
+    #[serde(rename = "matchedRules")]
+    matched_rules: &'a [RuleMatch],
+    #[serde(skip_serializing_if = "Option::is_none")]
+    decision: Option<Decision>,
+}

codex-rs/execpolicy/src/main.rs

@@ -1,6 +1,6 @@
 use anyhow::Result;
 use clap::Parser;
-use codex_execpolicy::ExecPolicyCheckCommand;
+use codex_execpolicy::execpolicycheck::ExecPolicyCheckCommand;
 
 /// CLI for evaluating exec policies
 #[derive(Parser)]
@@ -13,10 +13,6 @@ enum Cli {
 fn main() -> Result<()> {
     let cli = Cli::parse();
     match cli {
-        Cli::Check(cmd) => cmd_check(cmd),
+        Cli::Check(cmd) => cmd.run(),
     }
 }
-
-fn cmd_check(cmd: ExecPolicyCheckCommand) -> Result<()> {
-    cmd.run()
-}

codex-rs/execpolicy/src/policy.rs

@@ -11,6 +11,8 @@ use serde::Deserialize;
 use serde::Serialize;
 use std::sync::Arc;
 
+type HeuristicsFallback<'a> = Option<&'a dyn Fn(&[String]) -> Decision>;
+
 #[derive(Clone, Debug)]
 pub struct Policy {
     rules_by_program: MultiMap<String, RuleRef>,
@@ -50,62 +52,84 @@ impl Policy {
         Ok(())
     }
 
-    pub fn check(&self, cmd: &[String]) -> Evaluation {
-        let rules = match cmd.first() {
-            Some(first) => match self.rules_by_program.get_vec(first) {
-                Some(rules) => rules,
-                None => return Evaluation::NoMatch {},
-            },
-            None => return Evaluation::NoMatch {},
-        };
-
-        let matched_rules: Vec<RuleMatch> =
-            rules.iter().filter_map(|rule| rule.matches(cmd)).collect();
-        match matched_rules.iter().map(RuleMatch::decision).max() {
-            Some(decision) => Evaluation::Match {
-                decision,
-                matched_rules,
-            },
-            None => Evaluation::NoMatch {},
-        }
+    pub fn check<F>(&self, cmd: &[String], heuristics_fallback: &F) -> Evaluation
+    where
+        F: Fn(&[String]) -> Decision,
+    {
+        let matched_rules = self.matches_for_command(cmd, Some(heuristics_fallback));
+        Evaluation::from_matches(matched_rules)
     }
 
-    pub fn check_multiple<Commands>(&self, commands: Commands) -> Evaluation
+    pub fn check_multiple<Commands, F>(
+        &self,
+        commands: Commands,
+        heuristics_fallback: &F,
+    ) -> Evaluation
     where
         Commands: IntoIterator,
         Commands::Item: AsRef<[String]>,
+        F: Fn(&[String]) -> Decision,
     {
         let matched_rules: Vec<RuleMatch> = commands
             .into_iter()
-            .flat_map(|command| match self.check(command.as_ref()) {
-                Evaluation::Match { matched_rules, .. } => matched_rules,
-                Evaluation::NoMatch { .. } => Vec::new(),
+            .flat_map(|command| {
+                self.matches_for_command(command.as_ref(), Some(heuristics_fallback))
             })
             .collect();
 
-        match matched_rules.iter().map(RuleMatch::decision).max() {
-            Some(decision) => Evaluation::Match {
-                decision,
-                matched_rules,
-            },
-            None => Evaluation::NoMatch {},
+        Evaluation::from_matches(matched_rules)
+    }
+
+    pub fn matches_for_command(
+        &self,
+        cmd: &[String],
+        heuristics_fallback: HeuristicsFallback<'_>,
+    ) -> Vec<RuleMatch> {
+        let mut matched_rules: Vec<RuleMatch> = match cmd.first() {
+            Some(first) => self
+                .rules_by_program
+                .get_vec(first)
+                .map(|rules| rules.iter().filter_map(|rule| rule.matches(cmd)).collect())
+                .unwrap_or_default(),
+            None => Vec::new(),
+        };
+
+        if let (true, Some(heuristics_fallback)) = (matched_rules.is_empty(), heuristics_fallback) {
+            matched_rules.push(RuleMatch::HeuristicsRuleMatch {
+                command: cmd.to_vec(),
+                decision: heuristics_fallback(cmd),
+            });
         }
+
+        matched_rules
     }
 }
 
 #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub enum Evaluation {
-    NoMatch {},
-    Match {
-        decision: Decision,
-        #[serde(rename = "matchedRules")]
-        matched_rules: Vec<RuleMatch>,
-    },
+pub struct Evaluation {
+    pub decision: Decision,
+    #[serde(rename = "matchedRules")]
+    pub matched_rules: Vec<RuleMatch>,
 }
 
 impl Evaluation {
     pub fn is_match(&self) -> bool {
-        matches!(self, Self::Match { .. })
+        self.matched_rules
+            .iter()
+            .any(|rule_match| !matches!(rule_match, RuleMatch::HeuristicsRuleMatch { .. }))
+    }
+
+    fn from_matches(matched_rules: Vec<RuleMatch>) -> Self {
+        let decision = matched_rules
+            .iter()
+            .map(RuleMatch::decision)
+            .max()
+            .unwrap_or(Decision::Allow);
+
+        Self {
+            decision,
+            matched_rules,
+        }
     }
 }

codex-rs/execpolicy/src/rule.rs

@@ -64,12 +64,17 @@ pub enum RuleMatch {
         matched_prefix: Vec<String>,
         decision: Decision,
     },
+    HeuristicsRuleMatch {
+        command: Vec<String>,
+        decision: Decision,
+    },
 }
 
 impl RuleMatch {
     pub fn decision(&self) -> Decision {
         match self {
             Self::PrefixRuleMatch { decision, .. } => *decision,
+            Self::HeuristicsRuleMatch { decision, .. } => *decision,
         }
     }
 }