Add Object Storage

.env.example

@@ -0,0 +1,34 @@
+# Example environment configuration for CLIProxyAPI.
+# Copy this file to `.env` and uncomment the variables you need.
+#
+# NOTE: Environment variables are only required when using remote storage options.
+# For local file-based storage (default), no environment variables need to be set.
+
+# ------------------------------------------------------------------------------
+# Management Web UI
+# ------------------------------------------------------------------------------
+# MANAGEMENT_PASSWORD=change-me-to-a-strong-password
+
+# ------------------------------------------------------------------------------
+# Postgres Token Store (optional)
+# ------------------------------------------------------------------------------
+# PGSTORE_DSN=postgresql://user:pass@localhost:5432/cliproxy
+# PGSTORE_SCHEMA=public
+# PGSTORE_LOCAL_PATH=/var/lib/cliproxy
+
+# ------------------------------------------------------------------------------
+# Git-Backed Config Store (optional)
+# ------------------------------------------------------------------------------
+# GITSTORE_GIT_URL=https://github.com/your-org/cli-proxy-config.git
+# GITSTORE_GIT_USERNAME=git-user
+# GITSTORE_GIT_TOKEN=ghp_your_personal_access_token
+# GITSTORE_LOCAL_PATH=/data/cliproxy/gitstore
+
+# ------------------------------------------------------------------------------
+# Object Store Token Store (optional)
+# ------------------------------------------------------------------------------
+# OBJECTSTORE_ENDPOINT=https://s3.your-cloud.example.com
+# OBJECTSTORE_BUCKET=cli-proxy-config
+# OBJECTSTORE_ACCESS_KEY=your_access_key
+# OBJECTSTORE_SECRET_KEY=your_secret_key
+# OBJECTSTORE_LOCAL_PATH=/data/cliproxy/objectstore

.gitignore

@@ -1,20 +1,32 @@
+# Binaries
+cli-proxy-api
+*.exe
+
+# Configuration
 config.yaml
+.env
+
+# Generated content
 bin/*
-docs/*
 logs/*
 conv/*
+temp/*
+pgstore/*
+gitstore/*
+objectstore/*
+static/*
+
+# Authentication data
 auths/*
 !auths/.gitkeep
-.vscode/*
+
-.claude/*
+# Documentation
-.serena/*
+docs/*
 AGENTS.md
 CLAUDE.md
 GEMINI.md
-*.exe
+
-temp/*
+# Tooling metadata
-cli-proxy-api
+.vscode/*
-static/*
+.claude/*
-.env
+.serena/*
-pgstore/*
-gitstore/*

README.md

@@ -456,6 +456,27 @@ You can also persist configuration and authentication data in PostgreSQL when ru
 3.  **Bootstrapping:** If no configuration row exists, `config.example.yaml` seeds the database using the fixed identifier `config`.
 4.  **Token Sync:** Changes flow both ways—file updates are written to PostgreSQL and database records are mirrored back to disk so watchers and management APIs continue to operate.
 
+### Object Storage-backed Configuration and Token Store
+
+An S3-compatible object storage service can host configuration and authentication records.
+
+**Environment Variables**
+
+| Variable                 | Required | Default                        | Description                                                                                                              |
+|--------------------------|----------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------|
+| `OBJECTSTORE_ENDPOINT`   | Yes      |                                | Object storage endpoint. Include `http://` or `https://` to force the protocol (omitted scheme → HTTPS).                |
+| `OBJECTSTORE_BUCKET`     | Yes      |                                | Bucket that stores `config/config.yaml` and `auths/*.json`.                                                             |
+| `OBJECTSTORE_ACCESS_KEY` | Yes      |                                | Access key ID for the object storage account.                                                                           |
+| `OBJECTSTORE_SECRET_KEY` | Yes      |                                | Secret key for the object storage account.                                                                              |
+| `OBJECTSTORE_LOCAL_PATH` | No       | Current working directory      | Root directory for the local mirror; the server writes to `<value>/objectstore`. If unset, defaults to current CWD.     |
+
+**How it Works**
+
+1. **Startup:** The endpoint is parsed (respecting any scheme prefix), a MinIO-compatible client is created in path-style mode, and the bucket is created when missing.
+2. **Local Mirror:** A writable cache at `<OBJECTSTORE_LOCAL_PATH or CWD>/objectstore` mirrors `config/config.yaml` and `auths/`.
+3. **Bootstrapping:** When `config/config.yaml` is absent in the bucket, the server copies `config.example.yaml`, uploads it, and uses it as the initial configuration.
+4. **Sync:** Changes to configuration or auth files are uploaded to the bucket, and remote updates are mirrored back to disk, keeping watchers and management APIs in sync.
+
 ### OpenAI Compatibility Providers
 
 Configure upstream OpenAI-compatible providers (e.g., OpenRouter) via `openai-compatibility`.

README_CN.md

@@ -469,6 +469,27 @@ openai-compatibility:
 3.  **引导：** 若数据库中无配置记录，会使用 `config.example.yaml` 初始化，并以固定标识 `config` 写入。
 4.  **令牌同步：** 配置与令牌的更改会写入 PostgreSQL，同时数据库中的内容也会反向同步至本地镜像，便于文件监听与管理接口继续工作。
 
+### 对象存储驱动的配置与令牌存储
+
+可以选择使用 S3 兼容的对象存储来托管配置与鉴权数据。
+
+**环境变量**
+
+| 变量                     | 是否必填 | 默认值             | 说明                                                                                                                     |
+|--------------------------|----------|--------------------|--------------------------------------------------------------------------------------------------------------------------|
+| `OBJECTSTORE_ENDPOINT`   | 是       |                    | 对象存储访问端点。可带 `http://` 或 `https://` 前缀指定协议（省略则默认 HTTPS）。                                      |
+| `OBJECTSTORE_BUCKET`     | 是       |                    | 用于存放 `config/config.yaml` 与 `auths/*.json` 的 Bucket 名称。                                                        |
+| `OBJECTSTORE_ACCESS_KEY` | 是       |                    | 对象存储账号的访问密钥 ID。                                                                                              |
+| `OBJECTSTORE_SECRET_KEY` | 是       |                    | 对象存储账号的访问密钥 Secret。                                                                                          |
+| `OBJECTSTORE_LOCAL_PATH` | 否       | 当前工作目录 (CWD) | 本地镜像根目录；服务会写入到 `<值>/objectstore`。                                                                         |
+
+**工作流程**
+
+1. **启动阶段：** 解析端点地址（识别协议前缀），创建 MinIO 兼容客户端并使用 Path-Style 模式，如 Bucket 不存在会自动创建。
+2. **本地镜像：** 在 `<OBJECTSTORE_LOCAL_PATH 或当前工作目录>/objectstore` 维护可写缓存，同步 `config/config.yaml` 与 `auths/`。
+3. **初始化：** 若 Bucket 中缺少配置文件，将以 `config.example.yaml` 为模板生成 `config/config.yaml` 并上传。
+4. **双向同步：** 本地变更会上传到对象存储，同时远端对象也会拉回到本地，保证文件监听、管理 API 与 CLI 命令行为一致。
+
 ### OpenAI 兼容上游提供商
 
 通过 `openai-compatibility` 配置上游 OpenAI 兼容提供商（例如 OpenRouter）。

cmd/server/main.go

@@ -9,11 +9,13 @@ import (
 	"flag"
 	"fmt"
 	"io/fs"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
+	"github.com/joho/godotenv"
 	configaccess "github.com/router-for-me/CLIProxyAPI/v6/internal/access/config_access"
 	"github.com/router-for-me/CLIProxyAPI/v6/internal/cmd"
 	"github.com/router-for-me/CLIProxyAPI/v6/internal/config"
@@ -107,13 +109,20 @@ func main() {
 		pgStoreSchema        string
 		pgStoreLocalPath     string
 		pgStoreInst          *store.PostgresStore
-		gitStoreLocalPath string
 		useGitStore          bool
 		gitStoreRemoteURL    string
 		gitStoreUser         string
 		gitStorePassword     string
+		gitStoreLocalPath    string
 		gitStoreInst         *store.GitTokenStore
 		gitStoreRoot         string
+		useObjectStore       bool
+		objectStoreEndpoint  string
+		objectStoreAccess    string
+		objectStoreSecret    string
+		objectStoreBucket    string
+		objectStoreLocalPath string
+		objectStoreInst      *store.ObjectTokenStore
 	)
 
 	wd, err := os.Getwd()
@@ -121,6 +130,13 @@ func main() {
 		log.Fatalf("failed to get working directory: %v", err)
 	}
 
+	// Load environment variables from .env if present.
+	if errLoad := godotenv.Load(filepath.Join(wd, ".env")); errLoad != nil {
+		if !errors.Is(errLoad, os.ErrNotExist) {
+			log.WithError(errLoad).Warn("failed to load .env file")
+		}
+	}
+
 	lookupEnv := func(keys ...string) (string, bool) {
 		for _, key := range keys {
 			if value, ok := os.LookupEnv(key); ok {
@@ -157,6 +173,22 @@ func main() {
 	if value, ok := lookupEnv("GITSTORE_LOCAL_PATH", "gitstore_local_path"); ok {
 		gitStoreLocalPath = value
 	}
+	if value, ok := lookupEnv("OBJECTSTORE_ENDPOINT", "objectstore_endpoint"); ok {
+		useObjectStore = true
+		objectStoreEndpoint = value
+	}
+	if value, ok := lookupEnv("OBJECTSTORE_ACCESS_KEY", "objectstore_access_key"); ok {
+		objectStoreAccess = value
+	}
+	if value, ok := lookupEnv("OBJECTSTORE_SECRET_KEY", "objectstore_secret_key"); ok {
+		objectStoreSecret = value
+	}
+	if value, ok := lookupEnv("OBJECTSTORE_BUCKET", "objectstore_bucket"); ok {
+		objectStoreBucket = value
+	}
+	if value, ok := lookupEnv("OBJECTSTORE_LOCAL_PATH", "objectstore_local_path"); ok {
+		objectStoreLocalPath = value
+	}
 
 	// Check for cloud deploy mode only on first execution
 	// Read env var name in uppercase: DEPLOY
@@ -196,6 +228,65 @@ func main() {
 			cfg.AuthDir = pgStoreInst.AuthDir()
 			log.Infof("postgres-backed token store enabled, workspace path: %s", pgStoreInst.WorkDir())
 		}
+	} else if useObjectStore {
+		objectStoreRoot := objectStoreLocalPath
+		if objectStoreRoot == "" {
+			objectStoreRoot = wd
+		}
+		objectStoreRoot = filepath.Join(objectStoreRoot, "objectstore")
+		resolvedEndpoint := strings.TrimSpace(objectStoreEndpoint)
+		useSSL := true
+		if strings.Contains(resolvedEndpoint, "://") {
+			parsed, errParse := url.Parse(resolvedEndpoint)
+			if errParse != nil {
+				log.Fatalf("failed to parse object store endpoint %q: %v", objectStoreEndpoint, errParse)
+			}
+			switch strings.ToLower(parsed.Scheme) {
+			case "http":
+				useSSL = false
+			case "https":
+				useSSL = true
+			default:
+				log.Fatalf("unsupported object store scheme %q (only http and https are allowed)", parsed.Scheme)
+			}
+			if parsed.Host == "" {
+				log.Fatalf("object store endpoint %q is missing host information", objectStoreEndpoint)
+			}
+			resolvedEndpoint = parsed.Host
+			if parsed.Path != "" && parsed.Path != "/" {
+				resolvedEndpoint = strings.TrimSuffix(parsed.Host+parsed.Path, "/")
+			}
+		}
+		resolvedEndpoint = strings.TrimRight(resolvedEndpoint, "/")
+		objCfg := store.ObjectStoreConfig{
+			Endpoint:  resolvedEndpoint,
+			Bucket:    objectStoreBucket,
+			AccessKey: objectStoreAccess,
+			SecretKey: objectStoreSecret,
+			LocalRoot: objectStoreRoot,
+			UseSSL:    useSSL,
+			PathStyle: true,
+		}
+		objectStoreInst, err = store.NewObjectTokenStore(objCfg)
+		if err != nil {
+			log.Fatalf("failed to initialize object token store: %v", err)
+		}
+		examplePath := filepath.Join(wd, "config.example.yaml")
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		if errBootstrap := objectStoreInst.Bootstrap(ctx, examplePath); errBootstrap != nil {
+			cancel()
+			log.Fatalf("failed to bootstrap object-backed config: %v", errBootstrap)
+		}
+		cancel()
+		configFilePath = objectStoreInst.ConfigPath()
+		cfg, err = config.LoadConfigOptional(configFilePath, isCloudDeploy)
+		if err == nil {
+			if cfg == nil {
+				cfg = &config.Config{}
+			}
+			cfg.AuthDir = objectStoreInst.AuthDir()
+			log.Infof("object-backed token store enabled, bucket: %s", objectStoreBucket)
+		}
 	} else if useGitStore {
 		if gitStoreLocalPath == "" {
 			gitStoreLocalPath = wd
@@ -294,6 +385,8 @@ func main() {
 	// Register the shared token store once so all components use the same persistence backend.
 	if usePostgresStore {
 		sdkAuth.RegisterTokenStore(pgStoreInst)
+	} else if useObjectStore {
+		sdkAuth.RegisterTokenStore(objectStoreInst)
 	} else if useGitStore {
 		sdkAuth.RegisterTokenStore(gitStoreInst)
 	} else {

docker-compose.yml

@@ -10,6 +10,8 @@ services:
         COMMIT: ${COMMIT:-none}
         BUILD_DATE: ${BUILD_DATE:-unknown}
     container_name: cli-proxy-api
+    # env_file:
+    #   - .env
     environment:
       DEPLOY: ${DEPLOY:-}
     ports:

go.mod

@@ -7,8 +7,10 @@ require (
 	github.com/gin-gonic/gin v1.10.1
 	github.com/go-git/go-git/v6 v6.0.0-20251009132922-75a182125145
 	github.com/google/uuid v1.6.0
+	github.com/joho/godotenv v1.5.1
 	github.com/jackc/pgx/v5 v5.7.6
-	github.com/klauspost/compress v1.17.3
+	github.com/klauspost/compress v1.17.4
+	github.com/minio/minio-go/v7 v7.0.66
 	github.com/sirupsen/logrus v1.9.3
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/tidwall/gjson v1.18.0
@@ -30,6 +32,7 @@ require (
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
@@ -48,10 +51,13 @@ require (
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pjbgf/sha1cd v0.5.0 // indirect
+	github.com/rs/xid v1.5.0 // indirect
 	github.com/sergi/go-diff v1.4.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
@@ -62,4 +68,5 @@ require (
 	golang.org/x/sys v0.37.0 // indirect
 	golang.org/x/text v0.30.0 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 )

go.sum

@@ -23,6 +23,8 @@ github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGL
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
 github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -70,12 +72,17 @@ github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk=
 github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kevinburke/ssh_config v1.4.0 h1:6xxtP5bZ2E4NF5tuQulISpTO2z8XbtH8cg1PWkxoFkQ=
 github.com/kevinburke/ssh_config v1.4.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M=
 github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
 github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
+github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
@@ -90,6 +97,12 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
+github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs=
+github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
+github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -103,6 +116,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
 github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
@@ -163,6 +178,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

internal/store/objectstore.go

@@ -0,0 +1,618 @@
+package store
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/router-for-me/CLIProxyAPI/v6/internal/misc"
+	cliproxyauth "github.com/router-for-me/CLIProxyAPI/v6/sdk/cliproxy/auth"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	objectStoreConfigKey  = "config/config.yaml"
+	objectStoreAuthPrefix = "auths"
+)
+
+// ObjectStoreConfig captures configuration for the object storage-backed token store.
+type ObjectStoreConfig struct {
+	Endpoint  string
+	Bucket    string
+	AccessKey string
+	SecretKey string
+	Region    string
+	Prefix    string
+	LocalRoot string
+	UseSSL    bool
+	PathStyle bool
+}
+
+// ObjectTokenStore persists configuration and authentication metadata using an S3-compatible object storage backend.
+// Files are mirrored to a local workspace so existing file-based flows continue to operate.
+type ObjectTokenStore struct {
+	client     *minio.Client
+	cfg        ObjectStoreConfig
+	spoolRoot  string
+	configPath string
+	authDir    string
+	mu         sync.Mutex
+}
+
+// NewObjectTokenStore initializes an object storage backed token store.
+func NewObjectTokenStore(cfg ObjectStoreConfig) (*ObjectTokenStore, error) {
+	cfg.Endpoint = strings.TrimSpace(cfg.Endpoint)
+	cfg.Bucket = strings.TrimSpace(cfg.Bucket)
+	cfg.AccessKey = strings.TrimSpace(cfg.AccessKey)
+	cfg.SecretKey = strings.TrimSpace(cfg.SecretKey)
+	cfg.Prefix = strings.Trim(cfg.Prefix, "/")
+
+	if cfg.Endpoint == "" {
+		return nil, fmt.Errorf("object store: endpoint is required")
+	}
+	if cfg.Bucket == "" {
+		return nil, fmt.Errorf("object store: bucket is required")
+	}
+	if cfg.AccessKey == "" {
+		return nil, fmt.Errorf("object store: access key is required")
+	}
+	if cfg.SecretKey == "" {
+		return nil, fmt.Errorf("object store: secret key is required")
+	}
+
+	root := strings.TrimSpace(cfg.LocalRoot)
+	if root == "" {
+		if cwd, err := os.Getwd(); err == nil {
+			root = filepath.Join(cwd, "objectstore")
+		} else {
+			root = filepath.Join(os.TempDir(), "objectstore")
+		}
+	}
+	absRoot, err := filepath.Abs(root)
+	if err != nil {
+		return nil, fmt.Errorf("object store: resolve spool directory: %w", err)
+	}
+
+	configDir := filepath.Join(absRoot, "config")
+	authDir := filepath.Join(absRoot, "auths")
+
+	if err = os.MkdirAll(configDir, 0o700); err != nil {
+		return nil, fmt.Errorf("object store: create config directory: %w", err)
+	}
+	if err = os.MkdirAll(authDir, 0o700); err != nil {
+		return nil, fmt.Errorf("object store: create auth directory: %w", err)
+	}
+
+	options := &minio.Options{
+		Creds:  credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
+		Secure: cfg.UseSSL,
+		Region: cfg.Region,
+	}
+	if cfg.PathStyle {
+		options.BucketLookup = minio.BucketLookupPath
+	}
+
+	client, err := minio.New(cfg.Endpoint, options)
+	if err != nil {
+		return nil, fmt.Errorf("object store: create client: %w", err)
+	}
+
+	return &ObjectTokenStore{
+		client:     client,
+		cfg:        cfg,
+		spoolRoot:  absRoot,
+		configPath: filepath.Join(configDir, "config.yaml"),
+		authDir:    authDir,
+	}, nil
+}
+
+// SetBaseDir implements the optional interface used by authenticators; it is a no-op because
+// the object store controls its own workspace.
+func (s *ObjectTokenStore) SetBaseDir(string) {}
+
+// ConfigPath returns the managed configuration file path inside the spool directory.
+func (s *ObjectTokenStore) ConfigPath() string {
+	if s == nil {
+		return ""
+	}
+	return s.configPath
+}
+
+// AuthDir returns the local directory containing mirrored auth files.
+func (s *ObjectTokenStore) AuthDir() string {
+	if s == nil {
+		return ""
+	}
+	return s.authDir
+}
+
+// Bootstrap ensures the target bucket exists and synchronizes data from the object storage backend.
+func (s *ObjectTokenStore) Bootstrap(ctx context.Context, exampleConfigPath string) error {
+	if s == nil {
+		return fmt.Errorf("object store: not initialized")
+	}
+	if err := s.ensureBucket(ctx); err != nil {
+		return err
+	}
+	if err := s.syncConfigFromBucket(ctx, exampleConfigPath); err != nil {
+		return err
+	}
+	if err := s.syncAuthFromBucket(ctx); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Save persists authentication metadata to disk and uploads it to the object storage backend.
+func (s *ObjectTokenStore) Save(ctx context.Context, auth *cliproxyauth.Auth) (string, error) {
+	if auth == nil {
+		return "", fmt.Errorf("object store: auth is nil")
+	}
+
+	path, err := s.resolveAuthPath(auth)
+	if err != nil {
+		return "", err
+	}
+	if path == "" {
+		return "", fmt.Errorf("object store: missing file path attribute for %s", auth.ID)
+	}
+
+	if auth.Disabled {
+		if _, statErr := os.Stat(path); errors.Is(statErr, fs.ErrNotExist) {
+			return "", nil
+		}
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if err = os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return "", fmt.Errorf("object store: create auth directory: %w", err)
+	}
+
+	switch {
+	case auth.Storage != nil:
+		if err = auth.Storage.SaveTokenToFile(path); err != nil {
+			return "", err
+		}
+	case auth.Metadata != nil:
+		raw, errMarshal := json.Marshal(auth.Metadata)
+		if errMarshal != nil {
+			return "", fmt.Errorf("object store: marshal metadata: %w", errMarshal)
+		}
+		if existing, errRead := os.ReadFile(path); errRead == nil {
+			if jsonEqual(existing, raw) {
+				return path, nil
+			}
+		} else if errRead != nil && !errors.Is(errRead, fs.ErrNotExist) {
+			return "", fmt.Errorf("object store: read existing metadata: %w", errRead)
+		}
+		tmp := path + ".tmp"
+		if errWrite := os.WriteFile(tmp, raw, 0o600); errWrite != nil {
+			return "", fmt.Errorf("object store: write temp auth file: %w", errWrite)
+		}
+		if errRename := os.Rename(tmp, path); errRename != nil {
+			return "", fmt.Errorf("object store: rename auth file: %w", errRename)
+		}
+	default:
+		return "", fmt.Errorf("object store: nothing to persist for %s", auth.ID)
+	}
+
+	if auth.Attributes == nil {
+		auth.Attributes = make(map[string]string)
+	}
+	auth.Attributes["path"] = path
+
+	if strings.TrimSpace(auth.FileName) == "" {
+		auth.FileName = auth.ID
+	}
+
+	if err = s.uploadAuth(ctx, path); err != nil {
+		return "", err
+	}
+	return path, nil
+}
+
+// List enumerates auth JSON files from the mirrored workspace.
+func (s *ObjectTokenStore) List(_ context.Context) ([]*cliproxyauth.Auth, error) {
+	dir := strings.TrimSpace(s.AuthDir())
+	if dir == "" {
+		return nil, fmt.Errorf("object store: auth directory not configured")
+	}
+	entries := make([]*cliproxyauth.Auth, 0, 32)
+	err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		if d.IsDir() {
+			return nil
+		}
+		if !strings.HasSuffix(strings.ToLower(d.Name()), ".json") {
+			return nil
+		}
+		auth, err := s.readAuthFile(path, dir)
+		if err != nil {
+			log.WithError(err).Warnf("object store: skip auth %s", path)
+			return nil
+		}
+		if auth != nil {
+			entries = append(entries, auth)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("object store: walk auth directory: %w", err)
+	}
+	return entries, nil
+}
+
+// Delete removes an auth file locally and remotely.
+func (s *ObjectTokenStore) Delete(ctx context.Context, id string) error {
+	id = strings.TrimSpace(id)
+	if id == "" {
+		return fmt.Errorf("object store: id is empty")
+	}
+	path, err := s.resolveDeletePath(id)
+	if err != nil {
+		return err
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if err = os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("object store: delete auth file: %w", err)
+	}
+	if err = s.deleteAuthObject(ctx, path); err != nil {
+		return err
+	}
+	return nil
+}
+
+// PersistAuthFiles uploads the provided auth files to the object storage backend.
+func (s *ObjectTokenStore) PersistAuthFiles(ctx context.Context, _ string, paths ...string) error {
+	if len(paths) == 0 {
+		return nil
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	for _, p := range paths {
+		trimmed := strings.TrimSpace(p)
+		if trimmed == "" {
+			continue
+		}
+		abs := trimmed
+		if !filepath.IsAbs(abs) {
+			abs = filepath.Join(s.authDir, trimmed)
+		}
+		if err := s.uploadAuth(ctx, abs); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// PersistConfig uploads the local configuration file to the object storage backend.
+func (s *ObjectTokenStore) PersistConfig(ctx context.Context) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	data, err := os.ReadFile(s.configPath)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return s.deleteObject(ctx, objectStoreConfigKey)
+		}
+		return fmt.Errorf("object store: read config file: %w", err)
+	}
+	if len(data) == 0 {
+		return s.deleteObject(ctx, objectStoreConfigKey)
+	}
+	return s.putObject(ctx, objectStoreConfigKey, data, "application/x-yaml")
+}
+
+func (s *ObjectTokenStore) ensureBucket(ctx context.Context) error {
+	exists, err := s.client.BucketExists(ctx, s.cfg.Bucket)
+	if err != nil {
+		return fmt.Errorf("object store: check bucket: %w", err)
+	}
+	if exists {
+		return nil
+	}
+	if err = s.client.MakeBucket(ctx, s.cfg.Bucket, minio.MakeBucketOptions{Region: s.cfg.Region}); err != nil {
+		return fmt.Errorf("object store: create bucket: %w", err)
+	}
+	return nil
+}
+
+func (s *ObjectTokenStore) syncConfigFromBucket(ctx context.Context, example string) error {
+	key := s.prefixedKey(objectStoreConfigKey)
+	_, err := s.client.StatObject(ctx, s.cfg.Bucket, key, minio.StatObjectOptions{})
+	switch {
+	case err == nil:
+		object, errGet := s.client.GetObject(ctx, s.cfg.Bucket, key, minio.GetObjectOptions{})
+		if errGet != nil {
+			return fmt.Errorf("object store: fetch config: %w", errGet)
+		}
+		defer object.Close()
+		data, errRead := io.ReadAll(object)
+		if errRead != nil {
+			return fmt.Errorf("object store: read config: %w", errRead)
+		}
+		if errWrite := os.WriteFile(s.configPath, normalizeLineEndingsBytes(data), 0o600); errWrite != nil {
+			return fmt.Errorf("object store: write config: %w", errWrite)
+		}
+	case isObjectNotFound(err):
+		if _, statErr := os.Stat(s.configPath); errors.Is(statErr, fs.ErrNotExist) {
+			if example != "" {
+				if errCopy := misc.CopyConfigTemplate(example, s.configPath); errCopy != nil {
+					return fmt.Errorf("object store: copy example config: %w", errCopy)
+				}
+			} else {
+				if errCreate := os.MkdirAll(filepath.Dir(s.configPath), 0o700); errCreate != nil {
+					return fmt.Errorf("object store: prepare config directory: %w", errCreate)
+				}
+				if errWrite := os.WriteFile(s.configPath, []byte{}, 0o600); errWrite != nil {
+					return fmt.Errorf("object store: create empty config: %w", errWrite)
+				}
+			}
+		}
+		data, errRead := os.ReadFile(s.configPath)
+		if errRead != nil {
+			return fmt.Errorf("object store: read local config: %w", errRead)
+		}
+		if len(data) > 0 {
+			if errPut := s.putObject(ctx, objectStoreConfigKey, data, "application/x-yaml"); errPut != nil {
+				return errPut
+			}
+		}
+	default:
+		return fmt.Errorf("object store: stat config: %w", err)
+	}
+	return nil
+}
+
+func (s *ObjectTokenStore) syncAuthFromBucket(ctx context.Context) error {
+	if err := os.RemoveAll(s.authDir); err != nil {
+		return fmt.Errorf("object store: reset auth directory: %w", err)
+	}
+	if err := os.MkdirAll(s.authDir, 0o700); err != nil {
+		return fmt.Errorf("object store: recreate auth directory: %w", err)
+	}
+
+	prefix := s.prefixedKey(objectStoreAuthPrefix + "/")
+	objectCh := s.client.ListObjects(ctx, s.cfg.Bucket, minio.ListObjectsOptions{
+		Prefix:    prefix,
+		Recursive: true,
+	})
+	for object := range objectCh {
+		if object.Err != nil {
+			return fmt.Errorf("object store: list auth objects: %w", object.Err)
+		}
+		rel := strings.TrimPrefix(object.Key, prefix)
+		if rel == "" || strings.HasSuffix(rel, "/") {
+			continue
+		}
+		relPath := filepath.FromSlash(rel)
+		if filepath.IsAbs(relPath) {
+			log.WithField("key", object.Key).Warn("object store: skip auth outside mirror")
+			continue
+		}
+		cleanRel := filepath.Clean(relPath)
+		if cleanRel == "." || cleanRel == ".." || strings.HasPrefix(cleanRel, ".."+string(os.PathSeparator)) {
+			log.WithField("key", object.Key).Warn("object store: skip auth outside mirror")
+			continue
+		}
+		local := filepath.Join(s.authDir, cleanRel)
+		if err := os.MkdirAll(filepath.Dir(local), 0o700); err != nil {
+			return fmt.Errorf("object store: prepare auth subdir: %w", err)
+		}
+		reader, errGet := s.client.GetObject(ctx, s.cfg.Bucket, object.Key, minio.GetObjectOptions{})
+		if errGet != nil {
+			return fmt.Errorf("object store: download auth %s: %w", object.Key, errGet)
+		}
+		data, errRead := io.ReadAll(reader)
+		_ = reader.Close()
+		if errRead != nil {
+			return fmt.Errorf("object store: read auth %s: %w", object.Key, errRead)
+		}
+		if errWrite := os.WriteFile(local, data, 0o600); errWrite != nil {
+			return fmt.Errorf("object store: write auth %s: %w", local, errWrite)
+		}
+	}
+	return nil
+}
+
+func (s *ObjectTokenStore) uploadAuth(ctx context.Context, path string) error {
+	if path == "" {
+		return nil
+	}
+	rel, err := filepath.Rel(s.authDir, path)
+	if err != nil {
+		return fmt.Errorf("object store: resolve auth relative path: %w", err)
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return s.deleteAuthObject(ctx, path)
+		}
+		return fmt.Errorf("object store: read auth file: %w", err)
+	}
+	if len(data) == 0 {
+		return s.deleteAuthObject(ctx, path)
+	}
+	key := objectStoreAuthPrefix + "/" + filepath.ToSlash(rel)
+	return s.putObject(ctx, key, data, "application/json")
+}
+
+func (s *ObjectTokenStore) deleteAuthObject(ctx context.Context, path string) error {
+	if path == "" {
+		return nil
+	}
+	rel, err := filepath.Rel(s.authDir, path)
+	if err != nil {
+		return fmt.Errorf("object store: resolve auth relative path: %w", err)
+	}
+	key := objectStoreAuthPrefix + "/" + filepath.ToSlash(rel)
+	return s.deleteObject(ctx, key)
+}
+
+func (s *ObjectTokenStore) putObject(ctx context.Context, key string, data []byte, contentType string) error {
+	if len(data) == 0 {
+		return s.deleteObject(ctx, key)
+	}
+	fullKey := s.prefixedKey(key)
+	reader := bytes.NewReader(data)
+	_, err := s.client.PutObject(ctx, s.cfg.Bucket, fullKey, reader, int64(len(data)), minio.PutObjectOptions{
+		ContentType: contentType,
+	})
+	if err != nil {
+		return fmt.Errorf("object store: put object %s: %w", fullKey, err)
+	}
+	return nil
+}
+
+func (s *ObjectTokenStore) deleteObject(ctx context.Context, key string) error {
+	fullKey := s.prefixedKey(key)
+	err := s.client.RemoveObject(ctx, s.cfg.Bucket, fullKey, minio.RemoveObjectOptions{})
+	if err != nil {
+		if isObjectNotFound(err) {
+			return nil
+		}
+		return fmt.Errorf("object store: delete object %s: %w", fullKey, err)
+	}
+	return nil
+}
+
+func (s *ObjectTokenStore) prefixedKey(key string) string {
+	key = strings.TrimLeft(key, "/")
+	if s.cfg.Prefix == "" {
+		return key
+	}
+	return strings.TrimLeft(s.cfg.Prefix+"/"+key, "/")
+}
+
+func (s *ObjectTokenStore) resolveAuthPath(auth *cliproxyauth.Auth) (string, error) {
+	if auth == nil {
+		return "", fmt.Errorf("object store: auth is nil")
+	}
+	if auth.Attributes != nil {
+		if path := strings.TrimSpace(auth.Attributes["path"]); path != "" {
+			if filepath.IsAbs(path) {
+				return path, nil
+			}
+			return filepath.Join(s.authDir, path), nil
+		}
+	}
+	fileName := strings.TrimSpace(auth.FileName)
+	if fileName == "" {
+		fileName = strings.TrimSpace(auth.ID)
+	}
+	if fileName == "" {
+		return "", fmt.Errorf("object store: auth %s missing filename", auth.ID)
+	}
+	if !strings.HasSuffix(strings.ToLower(fileName), ".json") {
+		fileName += ".json"
+	}
+	return filepath.Join(s.authDir, fileName), nil
+}
+
+func (s *ObjectTokenStore) resolveDeletePath(id string) (string, error) {
+	id = strings.TrimSpace(id)
+	if id == "" {
+		return "", fmt.Errorf("object store: id is empty")
+	}
+	// Absolute paths are honored as-is; callers must ensure they point inside the mirror.
+	if filepath.IsAbs(id) {
+		return id, nil
+	}
+	// Treat any non-absolute id (including nested like "team/foo") as relative to the mirror authDir.
+	// Normalize separators and guard against path traversal.
+	clean := filepath.Clean(filepath.FromSlash(id))
+	if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) {
+		return "", fmt.Errorf("object store: invalid auth identifier %s", id)
+	}
+	// Ensure .json suffix.
+	if !strings.HasSuffix(strings.ToLower(clean), ".json") {
+		clean += ".json"
+	}
+	return filepath.Join(s.authDir, clean), nil
+}
+
+func (s *ObjectTokenStore) readAuthFile(path, baseDir string) (*cliproxyauth.Auth, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read file: %w", err)
+	}
+	if len(data) == 0 {
+		return nil, nil
+	}
+	metadata := make(map[string]any)
+	if err = json.Unmarshal(data, &metadata); err != nil {
+		return nil, fmt.Errorf("unmarshal auth json: %w", err)
+	}
+	provider := strings.TrimSpace(valueAsString(metadata["type"]))
+	if provider == "" {
+		provider = "unknown"
+	}
+	info, err := os.Stat(path)
+	if err != nil {
+		return nil, fmt.Errorf("stat auth file: %w", err)
+	}
+	rel, errRel := filepath.Rel(baseDir, path)
+	if errRel != nil {
+		rel = filepath.Base(path)
+	}
+	rel = normalizeAuthID(rel)
+	attr := map[string]string{"path": path}
+	if email := strings.TrimSpace(valueAsString(metadata["email"])); email != "" {
+		attr["email"] = email
+	}
+	auth := &cliproxyauth.Auth{
+		ID:               rel,
+		Provider:         provider,
+		FileName:         rel,
+		Label:            labelFor(metadata),
+		Status:           cliproxyauth.StatusActive,
+		Attributes:       attr,
+		Metadata:         metadata,
+		CreatedAt:        info.ModTime(),
+		UpdatedAt:        info.ModTime(),
+		LastRefreshedAt:  time.Time{},
+		NextRefreshAfter: time.Time{},
+	}
+	return auth, nil
+}
+
+func normalizeLineEndingsBytes(data []byte) []byte {
+	replaced := bytes.ReplaceAll(data, []byte{'\r', '\n'}, []byte{'\n'})
+	return bytes.ReplaceAll(replaced, []byte{'\r'}, []byte{'\n'})
+}
+
+func isObjectNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	resp := minio.ToErrorResponse(err)
+	if resp.StatusCode == http.StatusNotFound {
+		return true
+	}
+	switch resp.Code {
+	case "NoSuchKey", "NotFound", "NoSuchBucket":
+		return true
+	}
+	return false
+}