standalone websearch extension (#23823)

## Summary

Add the extension-backed standalone `web.run` tool so Codex can call the
standalone search endpoint through the `codex-api` search client and
return its encrypted output to Responses.

- gate the new tool behind `standalone_web_search`
- install the extension in the app-server thread registry and hide
hosted `web_search` when standalone search is enabled for OpenAI
providers so the two paths stay mutually exclusive
- build search context from persisted history using a small tail
heuristic: previous user message, assistant text between the last two
user turns capped at about 1k tokens, and current user message

## Test Plan

- `cargo test -p codex-web-search-extension`
- `cargo test -p codex-api`
- `cargo test -p codex-core
hosted_tools_follow_provider_auth_model_and_config_gates`

codex-rs/ext/web-search/src/extension.rs

@@ -0,0 +1,173 @@
+use std::sync::Arc;
+
+use codex_api::ApproximateLocation;
+use codex_api::LocationType;
+use codex_api::SearchContextSize;
+use codex_api::SearchFilters;
+use codex_api::SearchSettings;
+use codex_core::config::Config;
+use codex_extension_api::ConfigContributor;
+use codex_extension_api::ExtensionData;
+use codex_extension_api::ExtensionRegistryBuilder;
+use codex_extension_api::ThreadLifecycleContributor;
+use codex_extension_api::ThreadStartInput;
+use codex_extension_api::ToolContributor;
+use codex_features::Feature;
+use codex_login::AuthManager;
+use codex_model_provider::create_model_provider;
+use codex_model_provider_info::ModelProviderInfo;
+use codex_protocol::config_types::WebSearchContextSize;
+use codex_protocol::config_types::WebSearchMode;
+
+use crate::tool::WebSearchTool;
+
+#[derive(Clone)]
+struct WebSearchExtension {
+    auth_manager: Arc<AuthManager>,
+}
+
+#[derive(Clone)]
+struct WebSearchExtensionConfig {
+    enabled: bool,
+    provider: ModelProviderInfo,
+    settings: SearchSettings,
+}
+
+impl From<&Config> for WebSearchExtensionConfig {
+    fn from(config: &Config) -> Self {
+        let web_search_mode = config.web_search_mode.value();
+        Self {
+            enabled: config.features.enabled(Feature::StandaloneWebSearch)
+                && config.model_provider.is_openai()
+                && web_search_mode != WebSearchMode::Disabled,
+            provider: config.model_provider.clone(),
+            settings: search_settings(config, web_search_mode),
+        }
+    }
+}
+
+fn search_settings(config: &Config, web_search_mode: WebSearchMode) -> SearchSettings {
+    let web_search_config = config.web_search_config.as_ref();
+    SearchSettings {
+        user_location: web_search_config
+            .and_then(|config| config.user_location.as_ref())
+            .map(|location| ApproximateLocation {
+                r#type: LocationType::Approximate,
+                country: location.country.clone(),
+                region: location.region.clone(),
+                city: location.city.clone(),
+                timezone: location.timezone.clone(),
+            }),
+        search_context_size: web_search_config
+            .and_then(|config| config.search_context_size)
+            .map(|size| match size {
+                WebSearchContextSize::Low => SearchContextSize::Low,
+                WebSearchContextSize::Medium => SearchContextSize::Medium,
+                WebSearchContextSize::High => SearchContextSize::High,
+            }),
+        filters: web_search_config
+            .and_then(|config| config.filters.as_ref())
+            .map(|filters| SearchFilters {
+                allowed_domains: filters.allowed_domains.clone(),
+                blocked_domains: None,
+            }),
+        external_web_access: Some(match web_search_mode {
+            WebSearchMode::Live => true,
+            WebSearchMode::Cached | WebSearchMode::Disabled => false,
+        }),
+        ..Default::default()
+    }
+}
+
+#[async_trait::async_trait]
+impl ThreadLifecycleContributor<Config> for WebSearchExtension {
+    async fn on_thread_start(&self, input: ThreadStartInput<'_, Config>) {
+        input
+            .thread_store
+            .insert(WebSearchExtensionConfig::from(input.config));
+    }
+}
+
+impl ConfigContributor<Config> for WebSearchExtension {
+    fn on_config_changed(
+        &self,
+        _session_store: &ExtensionData,
+        thread_store: &ExtensionData,
+        _previous_config: &Config,
+        new_config: &Config,
+    ) {
+        thread_store.insert(WebSearchExtensionConfig::from(new_config));
+    }
+}
+
+impl ToolContributor for WebSearchExtension {
+    fn tools(
+        &self,
+        session_store: &ExtensionData,
+        thread_store: &ExtensionData,
+    ) -> Vec<Arc<dyn codex_extension_api::ToolExecutor<codex_extension_api::ToolCall>>> {
+        let Some(config) = thread_store.get::<WebSearchExtensionConfig>() else {
+            return Vec::new();
+        };
+        if !config.enabled {
+            return Vec::new();
+        }
+
+        vec![Arc::new(WebSearchTool {
+            session_id: session_store.level_id().to_string(),
+            provider: create_model_provider(
+                config.provider.clone(),
+                Some(self.auth_manager.clone()),
+            ),
+            settings: config.settings.clone(),
+        })]
+    }
+}
+
+pub fn install(registry: &mut ExtensionRegistryBuilder<Config>, auth_manager: Arc<AuthManager>) {
+    let extension = Arc::new(WebSearchExtension { auth_manager });
+    registry.thread_lifecycle_contributor(extension.clone());
+    registry.config_contributor(extension.clone());
+    registry.tool_contributor(extension);
+}
+
+#[cfg(test)]
+mod tests {
+    use codex_extension_api::ExtensionData;
+    use codex_extension_api::ExtensionRegistryBuilder;
+    use codex_extension_api::ToolName;
+    use codex_login::CodexAuth;
+    use codex_model_provider_info::ModelProviderInfo;
+    use pretty_assertions::assert_eq;
+
+    use super::AuthManager;
+    use super::Config;
+    use super::WebSearchExtensionConfig;
+    use super::install;
+
+    #[test]
+    fn installed_extension_contributes_web_run_when_enabled() {
+        let mut builder = ExtensionRegistryBuilder::<Config>::new();
+        install(
+            &mut builder,
+            AuthManager::from_auth_for_testing(CodexAuth::from_api_key("dummy")),
+        );
+        let registry = builder.build();
+        let session_store = ExtensionData::new("session");
+        let thread_store = ExtensionData::new("11111111-1111-4111-8111-111111111111");
+        thread_store.insert(WebSearchExtensionConfig {
+            enabled: true,
+            provider: ModelProviderInfo::create_openai_provider(/*base_url*/ None),
+            settings: Default::default(),
+        });
+
+        let tool_names = registry
+            .tool_contributors()
+            .iter()
+            .flat_map(|contributor| contributor.tools(&session_store, &thread_store))
+            .map(|tool| tool.tool_name())
+            .collect::<Vec<_>>();
+
+        assert_eq!(tool_names, vec![ToolName::namespaced("web", "run")]);
+    }
+}

codex-rs/ext/web-search/src/history.rs

@@ -0,0 +1,170 @@
+use codex_api::SearchInput;
+use codex_core::parse_turn_item;
+use codex_protocol::items::TurnItem;
+use codex_protocol::models::ContentItem;
+use codex_protocol::models::ResponseItem;
+use codex_tools::retain_tail_from_last_n_user_messages;
+use codex_tools::truncate_assistant_output_text_to_token_budget;
+
+const ASSISTANT_CONTEXT_TOKEN_LIMIT: usize = 1_000;
+const ASSISTANT_ROLE: &str = "assistant";
+const USER_ROLE: &str = "user";
+
+/// Builds the conversation tail for standalone web search.
+///
+/// The tail keeps the previous user text message, up to 1k tokens of assistant
+/// text that followed it, and the current user text message.
+pub(crate) fn recent_input(items: &[ResponseItem]) -> Option<SearchInput> {
+    let mut messages = Vec::new();
+    for item in items {
+        push_visible_message(&mut messages, item);
+    }
+
+    retain_tail_from_last_n_user_messages(&mut messages, /*user_message_count*/ 2);
+    truncate_assistant_output_text_to_token_budget(&mut messages, ASSISTANT_CONTEXT_TOKEN_LIMIT);
+    (!messages.is_empty()).then_some(SearchInput::Items(messages))
+}
+
+fn push_visible_message(messages: &mut Vec<ResponseItem>, item: &ResponseItem) {
+    match item {
+        ResponseItem::Message { role, .. } if role == ASSISTANT_ROLE => {
+            messages.push(item.clone());
+        }
+        ResponseItem::Message {
+            id,
+            role,
+            content,
+            phase,
+        } if role == USER_ROLE
+            && matches!(parse_turn_item(item), Some(TurnItem::UserMessage(_))) =>
+        {
+            let content = content
+                .iter()
+                .filter(|item| matches!(item, ContentItem::InputText { .. }))
+                .cloned()
+                .collect::<Vec<_>>();
+            if !content.is_empty() {
+                messages.push(ResponseItem::Message {
+                    id: id.clone(),
+                    role: role.clone(),
+                    content,
+                    phase: phase.clone(),
+                });
+            }
+        }
+        _ => {}
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use codex_api::SearchInput;
+    use codex_protocol::models::ContentItem;
+    use codex_protocol::models::ResponseItem;
+    use pretty_assertions::assert_eq;
+
+    use super::ASSISTANT_ROLE;
+    use super::USER_ROLE;
+    use super::recent_input;
+
+    fn message(role: &str, text: &str) -> ResponseItem {
+        ResponseItem::Message {
+            id: None,
+            role: role.to_string(),
+            content: vec![if role == ASSISTANT_ROLE {
+                ContentItem::OutputText {
+                    text: text.to_string(),
+                }
+            } else {
+                ContentItem::InputText {
+                    text: text.to_string(),
+                }
+            }],
+            phase: None,
+        }
+    }
+
+    #[test]
+    fn keeps_current_user_and_previous_visible_turn() {
+        let items = vec![
+            message("system", "system"),
+            message(USER_ROLE, "old user"),
+            message(ASSISTANT_ROLE, "old assistant"),
+            message(USER_ROLE, "previous user"),
+            ResponseItem::FunctionCall {
+                id: None,
+                name: "tool".to_string(),
+                namespace: None,
+                arguments: "{}".to_string(),
+                call_id: "call-1".to_string(),
+            },
+            message(ASSISTANT_ROLE, "previous assistant"),
+            message("developer", "developer"),
+            message(USER_ROLE, "current user"),
+            message(ASSISTANT_ROLE, "current commentary"),
+        ];
+
+        assert_eq!(
+            recent_input(&items),
+            Some(SearchInput::Items(vec![
+                message(USER_ROLE, "previous user"),
+                message(ASSISTANT_ROLE, "previous assistant"),
+                message(USER_ROLE, "current user"),
+            ]))
+        );
+    }
+
+    #[test]
+    fn keeps_only_text_from_recent_user_messages() {
+        let previous_user = ResponseItem::Message {
+            id: None,
+            role: USER_ROLE.to_string(),
+            content: vec![
+                ContentItem::InputText {
+                    text: "previous user".to_string(),
+                },
+                ContentItem::InputImage {
+                    image_url: "data:image/png;base64,image".to_string(),
+                    detail: None,
+                },
+            ],
+            phase: None,
+        };
+        let items = vec![
+            previous_user,
+            message(ASSISTANT_ROLE, "previous assistant"),
+            message(USER_ROLE, "current user"),
+        ];
+
+        assert_eq!(
+            recent_input(&items),
+            Some(SearchInput::Items(vec![
+                message(USER_ROLE, "previous user"),
+                message(ASSISTANT_ROLE, "previous assistant"),
+                message(USER_ROLE, "current user"),
+            ]))
+        );
+    }
+
+    #[test]
+    fn ignores_contextual_user_messages_when_selecting_recent_turns() {
+        let items = vec![
+            message(USER_ROLE, "previous user"),
+            message(ASSISTANT_ROLE, "previous assistant"),
+            message(
+                USER_ROLE,
+                "<environment_context>\n<cwd>/tmp</cwd>\n</environment_context>",
+            ),
+            message(USER_ROLE, "current user"),
+        ];
+
+        assert_eq!(
+            recent_input(&items),
+            Some(SearchInput::Items(vec![
+                message(USER_ROLE, "previous user"),
+                message(ASSISTANT_ROLE, "previous assistant"),
+                message(USER_ROLE, "current user"),
+            ]))
+        );
+    }
+}

codex-rs/ext/web-search/src/lib.rs

@@ -0,0 +1,7 @@
+mod extension;
+mod history;
+mod output;
+mod schema;
+mod tool;
+
+pub use extension::install;

codex-rs/ext/web-search/src/output.rs

@@ -0,0 +1,72 @@
+use codex_extension_api::ToolOutput;
+use codex_extension_api::ToolPayload;
+use codex_protocol::models::FunctionCallOutputContentItem;
+use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ResponseInputItem;
+
+pub(crate) struct EncryptedSearchOutput {
+    encrypted_output: String,
+}
+
+impl EncryptedSearchOutput {
+    pub(crate) fn new(encrypted_output: String) -> Self {
+        Self { encrypted_output }
+    }
+}
+
+impl ToolOutput for EncryptedSearchOutput {
+    fn log_preview(&self) -> String {
+        "[encrypted standalone web search output]".to_string()
+    }
+
+    fn success_for_logging(&self) -> bool {
+        true
+    }
+
+    fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem {
+        // TODO: Make standalone search honor memories.disable_on_external_context,
+        // as hosted web search does.
+        ResponseInputItem::FunctionCallOutput {
+            call_id: call_id.to_string(),
+            output: FunctionCallOutputPayload::from_content_items(vec![
+                FunctionCallOutputContentItem::EncryptedContent {
+                    encrypted_content: self.encrypted_output.clone(),
+                },
+            ]),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use codex_extension_api::ToolPayload;
+    use codex_protocol::models::FunctionCallOutputContentItem;
+    use codex_protocol::models::FunctionCallOutputPayload;
+    use codex_protocol::models::ResponseInputItem;
+    use pretty_assertions::assert_eq;
+
+    use super::EncryptedSearchOutput;
+    use super::ToolOutput;
+
+    #[test]
+    fn emits_encrypted_function_call_output() {
+        let output = EncryptedSearchOutput::new("encrypted-search-output".to_string());
+
+        assert_eq!(
+            output.to_response_item(
+                "call-1",
+                &ToolPayload::Function {
+                    arguments: "{}".to_string(),
+                },
+            ),
+            ResponseInputItem::FunctionCallOutput {
+                call_id: "call-1".to_string(),
+                output: FunctionCallOutputPayload::from_content_items(vec![
+                    FunctionCallOutputContentItem::EncryptedContent {
+                        encrypted_content: "encrypted-search-output".to_string(),
+                    },
+                ]),
+            }
+        );
+    }
+}

codex-rs/ext/web-search/src/schema.rs

@@ -0,0 +1,36 @@
+use codex_api::SearchCommands;
+use schemars::r#gen::SchemaSettings;
+use serde_json::Map;
+use serde_json::Value;
+
+pub(crate) fn commands_schema() -> Value {
+    let schema = SchemaSettings::draft2019_09()
+        .with(|settings| {
+            settings.inline_subschemas = true;
+            settings.option_add_null_type = false;
+        })
+        .into_generator()
+        .into_root_schema_for::<SearchCommands>();
+    let schema = match serde_json::to_value(schema) {
+        Ok(schema) => schema,
+        Err(err) => panic!("search commands schema should serialize: {err}"),
+    };
+    let Value::Object(mut schema) = schema else {
+        unreachable!("search commands schema must be an object");
+    };
+
+    let mut tool_schema = Map::new();
+    for key in [
+        "properties",
+        "required",
+        "type",
+        "additionalProperties",
+        "$defs",
+        "definitions",
+    ] {
+        if let Some(value) = schema.remove(key) {
+            tool_schema.insert(key.to_string(), value);
+        }
+    }
+    Value::Object(tool_schema)
+}

codex-rs/ext/web-search/src/tool.rs

@@ -0,0 +1,113 @@
+use codex_api::ReqwestTransport;
+use codex_api::SearchClient;
+use codex_api::SearchCommands;
+use codex_api::SearchRequest;
+use codex_api::SearchSettings;
+use codex_extension_api::FunctionCallError;
+use codex_extension_api::ResponsesApiTool;
+use codex_extension_api::ToolCall;
+use codex_extension_api::ToolExecutor;
+use codex_extension_api::ToolName;
+use codex_extension_api::ToolOutput;
+use codex_extension_api::ToolSpec;
+use codex_extension_api::parse_tool_input_schema;
+use codex_login::default_client::build_reqwest_client;
+use codex_model_provider::SharedModelProvider;
+use codex_tools::ResponsesApiNamespace;
+use codex_tools::ResponsesApiNamespaceTool;
+use codex_tools::ToolExposure;
+use codex_tools::default_namespace_description;
+use http::HeaderMap;
+
+use crate::history::recent_input;
+use crate::output::EncryptedSearchOutput;
+use crate::schema::commands_schema;
+
+const WEB_NAMESPACE: &str = "web";
+const RUN_TOOL_NAME: &str = "run";
+const WEB_RUN_DESCRIPTION: &str = include_str!("../web_run_description.md");
+
+pub(crate) struct WebSearchTool {
+    pub(crate) session_id: String,
+    pub(crate) provider: SharedModelProvider,
+    pub(crate) settings: SearchSettings,
+}
+
+#[async_trait::async_trait]
+impl ToolExecutor<ToolCall> for WebSearchTool {
+    fn tool_name(&self) -> ToolName {
+        ToolName::namespaced(WEB_NAMESPACE, RUN_TOOL_NAME)
+    }
+
+    fn spec(&self) -> ToolSpec {
+        let parameters = match parse_tool_input_schema(&commands_schema()) {
+            Ok(parameters) => parameters,
+            Err(err) => panic!("search command schema should parse: {err}"),
+        };
+
+        ToolSpec::Namespace(ResponsesApiNamespace {
+            name: WEB_NAMESPACE.to_string(),
+            description: default_namespace_description(WEB_NAMESPACE),
+            tools: vec![ResponsesApiNamespaceTool::Function(ResponsesApiTool {
+                name: RUN_TOOL_NAME.to_string(),
+                description: WEB_RUN_DESCRIPTION.to_string(),
+                strict: false,
+                parameters,
+                output_schema: None,
+                defer_loading: None,
+            })],
+        })
+    }
+
+    fn exposure(&self) -> ToolExposure {
+        ToolExposure::DirectModelOnly
+    }
+
+    async fn handle(&self, call: ToolCall) -> Result<Box<dyn ToolOutput>, FunctionCallError> {
+        let commands = parse_commands(&call)?;
+        let provider = self
+            .provider
+            .api_provider()
+            .await
+            .map_err(|err| FunctionCallError::Fatal(err.to_string()))?;
+        let auth = self
+            .provider
+            .api_auth()
+            .await
+            .map_err(|err| FunctionCallError::Fatal(err.to_string()))?;
+        let client = SearchClient::new(
+            ReqwestTransport::new(build_reqwest_client()),
+            provider,
+            auth,
+        );
+        let request = SearchRequest {
+            id: self.session_id.clone(),
+            model: None,
+            reasoning: None,
+            input: recent_input(call.conversation_history.items()),
+            commands: Some(commands),
+            settings: Some(self.settings.clone()),
+            max_output_tokens: Some(
+                u64::try_from(call.truncation_policy.token_budget()).unwrap_or(u64::MAX),
+            ),
+        };
+        let response = client
+            .search(&request, HeaderMap::new())
+            .await
+            .map_err(|err| FunctionCallError::Fatal(err.to_string()))?;
+
+        Ok(Box::new(EncryptedSearchOutput::new(
+            response.encrypted_output,
+        )))
+    }
+}
+
+fn parse_commands(call: &ToolCall) -> Result<SearchCommands, FunctionCallError> {
+    let arguments = call.function_arguments()?;
+    if arguments.trim().is_empty() {
+        return Ok(SearchCommands::default());
+    }
+
+    serde_json::from_str(arguments)
+        .map_err(|err| FunctionCallError::RespondToModel(err.to_string()))
+}