Fixed: #140 #133 #80

feat(translator): add token counting functionality for Gemini, Claude, and CLI

- Introduced `TokenCount` handling across various Codex translators (Gemini, Claude, CLI) with respective implementations.
- Added utility methods for token counting and formatting responses.
- Integrated `tiktoken-go/tokenizer` library for tokenization.
- Updated CodexExecutor with token counting logic to support multiple models including GPT-5 variants.
- Refined go.mod and go.sum to include new dependencies.

feat(runtime): add token counting functionality across executors

- Implemented token counting in OpenAICompatExecutor, QwenExecutor, and IFlowExecutor.
- Added utilities for token counting and response formatting using `tiktoken-go/tokenizer`.
- Integrated token counting into translators for Gemini, Claude, and Gemini CLI.
- Enhanced multiple model support, including GPT-5 variants, for token counting.

docs: update environment variable instructions for multi-model support

- Added details for setting `ANTHROPIC_DEFAULT_OPUS_MODEL`, `ANTHROPIC_DEFAULT_SONNET_MODEL`, and `ANTHROPIC_DEFAULT_HAIKU_MODEL` for version 2.x.x.
- Clarified usage of `ANTHROPIC_MODEL` and `ANTHROPIC_SMALL_FAST_MODEL` for version 1.x.x.
- Expanded examples for setting environment variables across different models including Gemini, GPT-5, Claude, and Qwen3.

internal/translator/codex/claude/codex_claude_response.go

@@ -354,3 +354,7 @@ func buildReverseMapFromClaudeOriginalShortToOriginal(original []byte) map[strin
 	}
 	return rev
 }
+
+func ClaudeTokenCount(ctx context.Context, count int64) string {
+	return fmt.Sprintf(`{"input_tokens":%d}`, count)
+}

internal/translator/codex/claude/init.go

@@ -12,8 +12,9 @@ func init() {
 		Codex,
 		ConvertClaudeRequestToCodex,
 		interfaces.TranslateResponse{
-			Stream:    ConvertCodexResponseToClaude,
-			NonStream: ConvertCodexResponseToClaudeNonStream,
+			Stream:     ConvertCodexResponseToClaude,
+			NonStream:  ConvertCodexResponseToClaudeNonStream,
+			TokenCount: ClaudeTokenCount,
 		},
 	)
 }

internal/translator/codex/gemini-cli/codex_gemini-cli_response.go

@@ -6,6 +6,7 @@ package geminiCLI
 
 import (
 	"context"
+	"fmt"
 
 	. "github.com/router-for-me/CLIProxyAPI/v6/internal/translator/codex/gemini"
 	"github.com/tidwall/sjson"
@@ -54,3 +55,7 @@ func ConvertCodexResponseToGeminiCLINonStream(ctx context.Context, modelName str
 	strJSON, _ = sjson.SetRaw(json, "response", strJSON)
 	return strJSON
 }
+
+func GeminiCLITokenCount(ctx context.Context, count int64) string {
+	return fmt.Sprintf(`{"totalTokens":%d,"promptTokensDetails":[{"modality":"TEXT","tokenCount":%d}]}`, count, count)
+}

internal/translator/codex/gemini-cli/init.go

@@ -12,8 +12,9 @@ func init() {
 		Codex,
 		ConvertGeminiCLIRequestToCodex,
 		interfaces.TranslateResponse{
-			Stream:    ConvertCodexResponseToGeminiCLI,
-			NonStream: ConvertCodexResponseToGeminiCLINonStream,
+			Stream:     ConvertCodexResponseToGeminiCLI,
+			NonStream:  ConvertCodexResponseToGeminiCLINonStream,
+			TokenCount: GeminiCLITokenCount,
 		},
 	)
 }

internal/translator/codex/gemini/codex_gemini_response.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/tidwall/gjson"
@@ -330,3 +331,7 @@ func mustMarshalJSON(v interface{}) string {
 	}
 	return string(data)
 }
+
+func GeminiTokenCount(ctx context.Context, count int64) string {
+	return fmt.Sprintf(`{"totalTokens":%d,"promptTokensDetails":[{"modality":"TEXT","tokenCount":%d}]}`, count, count)
+}

internal/translator/codex/gemini/init.go

@@ -12,8 +12,9 @@ func init() {
 		Codex,
 		ConvertGeminiRequestToCodex,
 		interfaces.TranslateResponse{
-			Stream:    ConvertCodexResponseToGemini,
-			NonStream: ConvertCodexResponseToGeminiNonStream,
+			Stream:     ConvertCodexResponseToGemini,
+			NonStream:  ConvertCodexResponseToGeminiNonStream,
+			TokenCount: GeminiTokenCount,
 		},
 	)
 }