Delete tiktoken-rs (#7018)

codex-rs/core/src/codex.rs

@@ -133,7 +133,6 @@ use codex_protocol::protocol::InitialHistory;
 use codex_protocol::user_input::UserInput;
 use codex_utils_readiness::Readiness;
 use codex_utils_readiness::ReadinessFlag;
-use codex_utils_tokenizer::warm_model_cache;
 
 /// The high-level interface to the Codex system.
 /// It operates as a queue pair where you send submissions and receive events.
@@ -555,9 +554,6 @@ impl Session {
         // Create the mutable state for the Session.
         let state = SessionState::new(session_configuration.clone());
 
-        // Warm the tokenizer cache for the session model without blocking startup.
-        warm_model_cache(&session_configuration.model);
-
         let services = SessionServices {
             mcp_connection_manager: Arc::new(RwLock::new(McpConnectionManager::default())),
             mcp_startup_cancellation_token: CancellationToken::new(),

codex-rs/core/src/config/edit.rs

@@ -4,7 +4,6 @@ use crate::config::types::Notice;
 use anyhow::Context;
 use codex_protocol::config_types::ReasoningEffort;
 use codex_protocol::config_types::TrustLevel;
-use codex_utils_tokenizer::warm_model_cache;
 use std::collections::BTreeMap;
 use std::path::Path;
 use std::path::PathBuf;
@@ -231,9 +230,6 @@ impl ConfigDocument {
     fn apply(&mut self, edit: &ConfigEdit) -> anyhow::Result<bool> {
         match edit {
             ConfigEdit::SetModel { model, effort } => Ok({
-                if let Some(model) = &model {
-                    warm_model_cache(model)
-                }
                 let mut mutated = false;
                 mutated |= self.write_profile_value(
                     &["model"],

codex-rs/core/src/context_manager/history.rs

@@ -1,13 +1,13 @@
 use crate::codex::TurnContext;
 use crate::context_manager::normalize;
 use crate::truncate::TruncationPolicy;
+use crate::truncate::approx_token_count;
 use crate::truncate::truncate_function_output_items_with_policy;
 use crate::truncate::truncate_text;
 use codex_protocol::models::FunctionCallOutputPayload;
 use codex_protocol::models::ResponseItem;
 use codex_protocol::protocol::TokenUsage;
 use codex_protocol::protocol::TokenUsageInfo;
-use codex_utils_tokenizer::Tokenizer;
 use std::ops::Deref;
 
 /// Transcript of conversation history
@@ -74,26 +74,21 @@ impl ContextManager {
         history
     }
 
-    // Estimate the number of tokens in the history. Return None if no tokenizer
-    // is available. This does not consider the reasoning traces.
-    // /!\ The value is a lower bound estimate and does not represent the exact
-    // context length.
+    // Estimate token usage using byte-based heuristics from the truncation helpers.
+    // This is a coarse lower bound, not a tokenizer-accurate count.
     pub(crate) fn estimate_token_count(&self, turn_context: &TurnContext) -> Option<i64> {
-        let model = turn_context.client.get_model();
-        let tokenizer = Tokenizer::for_model(model.as_str()).ok()?;
         let model_family = turn_context.client.get_model_family();
+        let base_tokens =
+            i64::try_from(approx_token_count(model_family.base_instructions.as_str()))
+                .unwrap_or(i64::MAX);
 
-        Some(
-            self.items
-                .iter()
-                .map(|item| {
-                    serde_json::to_string(&item)
-                        .map(|item| tokenizer.count(&item))
-                        .unwrap_or_default()
-                })
-                .sum::<i64>()
-                + tokenizer.count(model_family.base_instructions.as_str()),
-        )
+        let items_tokens = self.items.iter().fold(0i64, |acc, item| {
+            let serialized = serde_json::to_string(item).unwrap_or_default();
+            let item_tokens = i64::try_from(approx_token_count(&serialized)).unwrap_or(i64::MAX);
+            acc.saturating_add(item_tokens)
+        });
+
+        Some(base_tokens.saturating_add(items_tokens))
     }
 
     pub(crate) fn remove_first_item(&mut self) {