enable tool search over dynamic tools (#18263)

## Summary

- Normalize deferred MCP and dynamic tools into `ToolSearchEntry` values
before constructing `ToolSearchHandler`.
- Move the tool-search entry adapter out of `tools/handlers` and into
`tools/tool_search_entry.rs` so the handlers directory stays focused on
handlers.
- Keep `ToolSearchHandler` operating over one generic entry list for
BM25 search, namespace grouping, and per-bucket default limits.

## Why

Follow-up cleanup for #17849. The dynamic tool-search support made the
handler juggle source-specific MCP and dynamic tool lists, index
arithmetic, output conversion, and namespace emission. This keeps source
adaptation outside the handler so the search loop itself is smaller and
source-agnostic.

## Validation

- `just fmt`
- `cargo test -p codex-core tools::handlers::tool_search::tests`
- `git diff --check`
- `cargo test -p codex-core` currently fails in unrelated
`plugins::manager::tests::list_marketplaces_ignores_installed_roots_missing_from_config`;
rerunning that single test fails the same way at
`core/src/plugins/manager_tests.rs:1692`.

---------

Co-authored-by: pash <pash@openai.com>

codex-rs/core/src/tools/handlers/tool_search.rs

@@ -4,33 +4,31 @@ use crate::tools::context::ToolPayload;
 use crate::tools::context::ToolSearchOutput;
 use crate::tools::registry::ToolHandler;
 use crate::tools::registry::ToolKind;
+use crate::tools::tool_search_entry::ToolSearchEntry;
 use bm25::Document;
 use bm25::Language;
 use bm25::SearchEngine;
 use bm25::SearchEngineBuilder;
-use codex_mcp::ToolInfo;
 use codex_tools::TOOL_SEARCH_DEFAULT_LIMIT;
 use codex_tools::TOOL_SEARCH_TOOL_NAME;
-use codex_tools::ToolSearchResultSource;
-use codex_tools::collect_tool_search_output_tools;
+use codex_tools::ToolSearchOutputTool;
+use std::collections::HashMap;
 
 const COMPUTER_USE_MCP_SERVER_NAME: &str = "computer-use";
 const COMPUTER_USE_TOOL_SEARCH_LIMIT: usize = 20;
 
 pub struct ToolSearchHandler {
-    entries: Vec<(String, ToolInfo)>,
+    entries: Vec<ToolSearchEntry>,
     search_engine: SearchEngine<usize>,
 }
 
 impl ToolSearchHandler {
-    pub fn new(tools: std::collections::HashMap<String, ToolInfo>) -> Self {
-        let mut entries: Vec<(String, ToolInfo)> = tools.into_iter().collect();
-        entries.sort_by(|a, b| a.0.cmp(&b.0));
-
+    pub(crate) fn new(entries: Vec<ToolSearchEntry>) -> Self {
         let documents: Vec<Document<usize>> = entries
             .iter()
+            .map(|entry| entry.search_text.clone())
             .enumerate()
-            .map(|(idx, (name, info))| Document::new(idx, build_search_text(name, info)))
+            .map(|(idx, search_text)| Document::new(idx, search_text))
             .collect();
         let search_engine =
             SearchEngineBuilder::<usize>::with_documents(Language::English, documents).build();
@@ -83,155 +81,230 @@ impl ToolHandler for ToolSearchHandler {
             return Ok(ToolSearchOutput { tools: Vec::new() });
         }
 
-        let results = self.search_result_entries(query, limit, requested_limit.is_none());
-
-        let tools = collect_tool_search_output_tools(results.into_iter().map(|(_, tool)| {
-            ToolSearchResultSource {
-                server_name: tool.server_name.as_str(),
-                tool_namespace: tool.callable_namespace.as_str(),
-                tool_name: tool.callable_name.as_str(),
-                tool: &tool.tool,
-                connector_name: tool.connector_name.as_deref(),
-                connector_description: tool.connector_description.as_deref(),
-            }
-        }))
-        .map_err(|err| {
-            FunctionCallError::Fatal(format!(
-                "failed to encode {TOOL_SEARCH_TOOL_NAME} output: {err}"
-            ))
-        })?;
+        let tools = self.search(query, limit, requested_limit.is_none())?;
 
         Ok(ToolSearchOutput { tools })
     }
 }
 
 impl ToolSearchHandler {
+    fn search(
+        &self,
+        query: &str,
+        limit: usize,
+        use_default_limit: bool,
+    ) -> Result<Vec<ToolSearchOutputTool>, FunctionCallError> {
+        let results = self.search_result_entries(query, limit, use_default_limit);
+        self.search_output_tools(results)
+    }
+
     fn search_result_entries(
         &self,
         query: &str,
         limit: usize,
         use_default_limit: bool,
-    ) -> Vec<&(String, ToolInfo)> {
+    ) -> Vec<&ToolSearchEntry> {
         let mut results = self
             .search_engine
             .search(query, limit)
             .into_iter()
-            .filter_map(|result| self.entries.get(result.document.id))
+            .map(|result| result.document.id)
+            .filter_map(|id| self.entries.get(id))
             .collect::<Vec<_>>();
         if !use_default_limit {
             return results;
         }
 
-        if results
-            .iter()
-            .any(|(_, tool)| tool.server_name == COMPUTER_USE_MCP_SERVER_NAME)
-        {
+        if results.iter().any(|entry| {
+            entry
+                .limit_bucket
+                .as_deref()
+                .is_some_and(|bucket| bucket == COMPUTER_USE_MCP_SERVER_NAME)
+        }) {
             results = self
                 .search_engine
                 .search(query, COMPUTER_USE_TOOL_SEARCH_LIMIT)
                 .into_iter()
-                .filter_map(|result| self.entries.get(result.document.id))
+                .map(|result| result.document.id)
+                .filter_map(|id| self.entries.get(id))
                 .collect();
         }
-        limit_results_per_server(results)
+        limit_results_by_bucket(results)
+    }
+
+    fn search_output_tools<'a>(
+        &self,
+        results: impl IntoIterator<Item = &'a ToolSearchEntry>,
+    ) -> Result<Vec<ToolSearchOutputTool>, FunctionCallError> {
+        let mut tools = Vec::new();
+        // Preserve search order: group namespace children, emit standalone tools directly.
+        for entry in results {
+            match &entry.output {
+                ToolSearchOutputTool::Function(tool) => {
+                    tools.push(ToolSearchOutputTool::Function(tool.clone()));
+                }
+                ToolSearchOutputTool::Namespace(namespace) => {
+                    if let Some(output) = tools.iter_mut().find_map(|tool| match tool {
+                        ToolSearchOutputTool::Namespace(output)
+                            if output.name == namespace.name =>
+                        {
+                            Some(output)
+                        }
+                        ToolSearchOutputTool::Namespace(_) | ToolSearchOutputTool::Function(_) => {
+                            None
+                        }
+                    }) {
+                        output.tools.extend(namespace.tools.clone());
+                    } else {
+                        tools.push(ToolSearchOutputTool::Namespace(namespace.clone()));
+                    }
+                }
+            }
+        }
+
+        Ok(tools)
     }
 }
 
-fn limit_results_per_server(results: Vec<&(String, ToolInfo)>) -> Vec<&(String, ToolInfo)> {
+fn limit_results_by_bucket(results: Vec<&ToolSearchEntry>) -> Vec<&ToolSearchEntry> {
     results
         .into_iter()
-        .scan(
-            std::collections::HashMap::<&str, usize>::new(),
-            |counts, entry| {
-                let tool = &entry.1;
-                let count = counts.entry(tool.server_name.as_str()).or_default();
-                if *count >= default_limit_for_server(tool.server_name.as_str()) {
-                    Some(None)
-                } else {
-                    *count += 1;
-                    Some(Some(entry))
-                }
-            },
-        )
+        .scan(HashMap::<&str, usize>::new(), |counts, result| {
+            let Some(bucket) = result.limit_bucket.as_deref() else {
+                return Some(Some(result));
+            };
+            let count = counts.entry(bucket).or_default();
+            if *count >= default_limit_for_bucket(bucket) {
+                Some(None)
+            } else {
+                *count += 1;
+                Some(Some(result))
+            }
+        })
         .flatten()
         .collect()
 }
 
-fn default_limit_for_server(server_name: &str) -> usize {
-    if server_name == COMPUTER_USE_MCP_SERVER_NAME {
+fn default_limit_for_bucket(bucket: &str) -> usize {
+    if bucket == COMPUTER_USE_MCP_SERVER_NAME {
         COMPUTER_USE_TOOL_SEARCH_LIMIT
     } else {
         TOOL_SEARCH_DEFAULT_LIMIT
     }
 }
 
-fn build_search_text(name: &str, info: &ToolInfo) -> String {
-    let mut parts = vec![
-        name.to_string(),
-        info.callable_name.clone(),
-        info.tool.name.to_string(),
-        info.server_name.clone(),
-    ];
-
-    if let Some(title) = info.tool.title.as_deref()
-        && !title.trim().is_empty()
-    {
-        parts.push(title.to_string());
-    }
-
-    if let Some(description) = info.tool.description.as_deref()
-        && !description.trim().is_empty()
-    {
-        parts.push(description.to_string());
-    }
-
-    if let Some(connector_name) = info.connector_name.as_deref()
-        && !connector_name.trim().is_empty()
-    {
-        parts.push(connector_name.to_string());
-    }
-
-    if let Some(connector_description) = info.connector_description.as_deref()
-        && !connector_description.trim().is_empty()
-    {
-        parts.push(connector_description.to_string());
-    }
-
-    parts.extend(
-        info.plugin_display_names
-            .iter()
-            .map(String::as_str)
-            .map(str::trim)
-            .filter(|name| !name.is_empty())
-            .map(str::to_string),
-    );
-
-    parts.extend(
-        info.tool
-            .input_schema
-            .get("properties")
-            .and_then(serde_json::Value::as_object)
-            .map(|map| map.keys().cloned().collect::<Vec<_>>())
-            .unwrap_or_default(),
-    );
-
-    parts.join(" ")
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::tools::tool_search_entry::build_tool_search_entries;
+    use codex_mcp::ToolInfo;
+    use codex_protocol::dynamic_tools::DynamicToolSpec;
+    use codex_tools::ResponsesApiNamespace;
+    use codex_tools::ResponsesApiNamespaceTool;
+    use codex_tools::ResponsesApiTool;
     use pretty_assertions::assert_eq;
     use rmcp::model::Tool;
     use std::sync::Arc;
 
+    #[test]
+    fn mixed_search_results_coalesce_mcp_namespaces() {
+        let dynamic_tools = vec![DynamicToolSpec {
+            name: "automation_update".to_string(),
+            description: "Create, update, view, or delete recurring automations.".to_string(),
+            input_schema: serde_json::json!({
+                "type": "object",
+                "properties": {
+                    "mode": { "type": "string" },
+                },
+                "required": ["mode"],
+                "additionalProperties": false,
+            }),
+            defer_loading: true,
+        }];
+        let handler = handler_from_tools(
+            Some(&std::collections::HashMap::from([
+                (
+                    "mcp__calendar__create_event".to_string(),
+                    tool_info("calendar", "create_event", "Create events"),
+                ),
+                (
+                    "mcp__calendar__list_events".to_string(),
+                    tool_info("calendar", "list_events", "List events"),
+                ),
+            ])),
+            &dynamic_tools,
+        );
+        let results = [
+            &handler.entries[0],
+            &handler.entries[2],
+            &handler.entries[1],
+        ];
+
+        let tools = handler
+            .search_output_tools(results)
+            .expect("mixed search output should serialize");
+
+        assert_eq!(
+            tools,
+            vec![
+                ToolSearchOutputTool::Namespace(ResponsesApiNamespace {
+                    name: "mcp__calendar__".to_string(),
+                    description: "Tools in the mcp__calendar__ namespace.".to_string(),
+                    tools: vec![
+                        ResponsesApiNamespaceTool::Function(ResponsesApiTool {
+                            name: "create_event".to_string(),
+                            description: "Create events desktop tool".to_string(),
+                            strict: false,
+                            defer_loading: Some(true),
+                            parameters: codex_tools::JsonSchema::object(
+                                Default::default(),
+                                /*required*/ None,
+                                Some(false.into()),
+                            ),
+                            output_schema: None,
+                        }),
+                        ResponsesApiNamespaceTool::Function(ResponsesApiTool {
+                            name: "list_events".to_string(),
+                            description: "List events desktop tool".to_string(),
+                            strict: false,
+                            defer_loading: Some(true),
+                            parameters: codex_tools::JsonSchema::object(
+                                Default::default(),
+                                /*required*/ None,
+                                Some(false.into()),
+                            ),
+                            output_schema: None,
+                        }),
+                    ],
+                }),
+                ToolSearchOutputTool::Function(ResponsesApiTool {
+                    name: "automation_update".to_string(),
+                    description: "Create, update, view, or delete recurring automations."
+                        .to_string(),
+                    strict: false,
+                    defer_loading: Some(true),
+                    parameters: codex_tools::JsonSchema::object(
+                        std::collections::BTreeMap::from([(
+                            "mode".to_string(),
+                            codex_tools::JsonSchema::string(/*description*/ None),
+                        )]),
+                        Some(vec!["mode".to_string()]),
+                        Some(false.into()),
+                    ),
+                    output_schema: None,
+                }),
+            ],
+        );
+    }
+
     #[test]
     fn computer_use_tool_search_uses_larger_limit() {
-        let handler = ToolSearchHandler::new(numbered_tools(
+        let tools = numbered_tools(
             COMPUTER_USE_MCP_SERVER_NAME,
             "computer use",
             /*count*/ 100,
-        ));
+        );
+        let handler = handler_from_tools(Some(&tools), &[]);
 
         let results = handler.search_result_entries(
             "computer use",
@@ -243,7 +316,7 @@ mod tests {
         assert!(
             results
                 .iter()
-                .all(|(_, tool)| tool.server_name == COMPUTER_USE_MCP_SERVER_NAME)
+                .all(|entry| entry.limit_bucket.as_deref() == Some(COMPUTER_USE_MCP_SERVER_NAME))
         );
 
         let explicit_results = handler.search_result_entries(
@@ -267,7 +340,7 @@ mod tests {
             "calendar",
             /*count*/ 100,
         ));
-        let handler = ToolSearchHandler::new(tools);
+        let handler = handler_from_tools(Some(&tools), &[]);
 
         let results = handler.search_result_entries(
             "calendar",
@@ -279,7 +352,7 @@ mod tests {
         assert!(
             results
                 .iter()
-                .all(|(_, tool)| tool.server_name == "other-server")
+                .all(|entry| entry.limit_bucket.as_deref() == Some("other-server"))
         );
 
         let explicit_results = handler.search_result_entries(
@@ -301,7 +374,7 @@ mod tests {
             "computer use",
             /*count*/ 100,
         ));
-        let handler = ToolSearchHandler::new(tools);
+        let handler = handler_from_tools(Some(&tools), &[]);
 
         let results = handler.search_result_entries(
             "computer use",
@@ -360,10 +433,17 @@ mod tests {
         }
     }
 
-    fn count_results_for_server(results: &[&(String, ToolInfo)], server_name: &str) -> usize {
+    fn count_results_for_server(results: &[&ToolSearchEntry], server_name: &str) -> usize {
         results
             .iter()
-            .filter(|(_, tool)| tool.server_name == server_name)
+            .filter(|entry| entry.limit_bucket.as_deref() == Some(server_name))
             .count()
     }
+
+    fn handler_from_tools(
+        mcp_tools: Option<&std::collections::HashMap<String, ToolInfo>>,
+        dynamic_tools: &[DynamicToolSpec],
+    ) -> ToolSearchHandler {
+        ToolSearchHandler::new(build_tool_search_entries(mcp_tools, dynamic_tools))
+    }
 }

codex-rs/core/src/tools/mod.rs

@@ -11,6 +11,7 @@ pub(crate) mod router;
 pub(crate) mod runtimes;
 pub(crate) mod sandboxing;
 pub(crate) mod spec;
+pub(crate) mod tool_search_entry;
 
 use codex_protocol::exec_output::ExecToolCallOutput;
 use codex_utils_output_truncation::TruncationPolicy;

codex-rs/core/src/tools/spec.rs

@@ -108,6 +108,7 @@ pub(crate) fn build_specs_with_discoverable_tools(
     use crate::tools::handlers::multi_agents_v2::SpawnAgentHandler as SpawnAgentHandlerV2;
     use crate::tools::handlers::multi_agents_v2::WaitAgentHandler as WaitAgentHandlerV2;
     use crate::tools::handlers::unavailable_tool_message;
+    use crate::tools::tool_search_entry::build_tool_search_entries;
 
     let mut builder = ToolRegistryBuilder::new();
     let mcp_tool_plan_inputs = mcp_tools.as_ref().map(map_mcp_tools_for_plan);
@@ -157,6 +158,11 @@ pub(crate) fn build_specs_with_discoverable_tools(
     let request_user_input_handler = Arc::new(RequestUserInputHandler {
         default_mode_request_user_input: config.default_mode_request_user_input,
     });
+    let deferred_dynamic_tools = dynamic_tools
+        .iter()
+        .filter(|tool| tool.defer_loading)
+        .cloned()
+        .collect::<Vec<_>>();
     let mut tool_search_handler = None;
     let tool_suggest_handler = Arc::new(ToolSuggestHandler);
     let code_mode_handler = Arc::new(CodeModeExecuteHandler);
@@ -259,9 +265,11 @@ pub(crate) fn build_specs_with_discoverable_tools(
             }
             ToolHandlerKind::ToolSearch => {
                 if tool_search_handler.is_none() {
-                    tool_search_handler = deferred_mcp_tools
-                        .as_ref()
-                        .map(|tools| Arc::new(ToolSearchHandler::new(tools.clone())));
+                    let entries = build_tool_search_entries(
+                        deferred_mcp_tools.as_ref(),
+                        &deferred_dynamic_tools,
+                    );
+                    tool_search_handler = Some(Arc::new(ToolSearchHandler::new(entries)));
                 }
                 if let Some(tool_search_handler) = tool_search_handler.as_ref() {
                     builder.register_handler(handler.name, tool_search_handler.clone());

codex-rs/core/src/tools/tool_search_entry.rs

@@ -0,0 +1,147 @@
+use codex_mcp::ToolInfo;
+use codex_protocol::dynamic_tools::DynamicToolSpec;
+use codex_tools::ToolSearchOutputTool;
+use codex_tools::ToolSearchResultSource;
+use codex_tools::dynamic_tool_to_responses_api_tool;
+use codex_tools::tool_search_result_source_to_output_tool;
+use std::collections::HashMap;
+
+#[derive(Clone)]
+pub(crate) struct ToolSearchEntry {
+    pub(crate) search_text: String,
+    pub(crate) output: ToolSearchOutputTool,
+    pub(crate) limit_bucket: Option<String>,
+}
+
+pub(crate) fn build_tool_search_entries(
+    mcp_tools: Option<&HashMap<String, ToolInfo>>,
+    dynamic_tools: &[DynamicToolSpec],
+) -> Vec<ToolSearchEntry> {
+    let mut entries = Vec::new();
+
+    let mut mcp_tools = mcp_tools
+        .map(|tools| tools.values().collect::<Vec<_>>())
+        .unwrap_or_default();
+    mcp_tools.sort_by_key(|info| info.canonical_tool_name().display());
+    for info in mcp_tools {
+        match mcp_tool_search_entry(info) {
+            Ok(entry) => entries.push(entry),
+            Err(error) => {
+                let tool_name = info.canonical_tool_name();
+                tracing::error!(
+                    "Failed to convert deferred MCP tool `{tool_name}` to OpenAI tool: {error:?}"
+                );
+            }
+        }
+    }
+
+    let mut dynamic_tools = dynamic_tools.iter().collect::<Vec<_>>();
+    dynamic_tools.sort_by(|a, b| a.name.cmp(&b.name));
+    for tool in dynamic_tools {
+        match dynamic_tool_search_entry(tool) {
+            Ok(entry) => entries.push(entry),
+            Err(error) => {
+                tracing::error!(
+                    "Failed to convert deferred dynamic tool {:?} to OpenAI tool: {error:?}",
+                    tool.name
+                );
+            }
+        }
+    }
+
+    entries
+}
+
+fn mcp_tool_search_entry(info: &ToolInfo) -> Result<ToolSearchEntry, serde_json::Error> {
+    Ok(ToolSearchEntry {
+        search_text: build_mcp_search_text(info),
+        output: tool_search_result_source_to_output_tool(ToolSearchResultSource {
+            server_name: info.server_name.as_str(),
+            tool_namespace: info.callable_namespace.as_str(),
+            tool_name: info.callable_name.as_str(),
+            tool: &info.tool,
+            connector_name: info.connector_name.as_deref(),
+            connector_description: info.connector_description.as_deref(),
+        })?,
+        limit_bucket: Some(info.server_name.clone()),
+    })
+}
+
+fn dynamic_tool_search_entry(tool: &DynamicToolSpec) -> Result<ToolSearchEntry, serde_json::Error> {
+    Ok(ToolSearchEntry {
+        search_text: build_dynamic_search_text(tool),
+        output: ToolSearchOutputTool::Function(dynamic_tool_to_responses_api_tool(tool)?),
+        limit_bucket: None,
+    })
+}
+
+fn build_mcp_search_text(info: &ToolInfo) -> String {
+    let mut parts = vec![
+        info.canonical_tool_name().display(),
+        info.callable_name.clone(),
+        info.tool.name.to_string(),
+        info.server_name.clone(),
+    ];
+
+    if let Some(title) = info.tool.title.as_deref()
+        && !title.trim().is_empty()
+    {
+        parts.push(title.to_string());
+    }
+
+    if let Some(description) = info.tool.description.as_deref()
+        && !description.trim().is_empty()
+    {
+        parts.push(description.to_string());
+    }
+
+    if let Some(connector_name) = info.connector_name.as_deref()
+        && !connector_name.trim().is_empty()
+    {
+        parts.push(connector_name.to_string());
+    }
+
+    if let Some(connector_description) = info.connector_description.as_deref()
+        && !connector_description.trim().is_empty()
+    {
+        parts.push(connector_description.to_string());
+    }
+
+    parts.extend(
+        info.plugin_display_names
+            .iter()
+            .map(String::as_str)
+            .map(str::trim)
+            .filter(|name| !name.is_empty())
+            .map(str::to_string),
+    );
+
+    parts.extend(
+        info.tool
+            .input_schema
+            .get("properties")
+            .and_then(serde_json::Value::as_object)
+            .map(|map| map.keys().cloned().collect::<Vec<_>>())
+            .unwrap_or_default(),
+    );
+
+    parts.join(" ")
+}
+
+fn build_dynamic_search_text(tool: &DynamicToolSpec) -> String {
+    let mut parts = vec![
+        tool.name.clone(),
+        tool.name.replace('_', " "),
+        tool.description.clone(),
+    ];
+
+    parts.extend(
+        tool.input_schema
+            .get("properties")
+            .and_then(serde_json::Value::as_object)
+            .map(|map| map.keys().cloned().collect::<Vec<_>>())
+            .unwrap_or_default(),
+    );
+
+    parts.join(" ")
+}