v6 version first commit

This commit is contained in:
Luis Pater
2025-09-22 01:40:24 +08:00
parent d42384cdb7
commit 4999fce7f4
171 changed files with 7626 additions and 7494 deletions

View File

@@ -9,6 +9,8 @@ import (
"path/filepath"
"strings"
"time"
bolt "go.etcd.io/bbolt"
)
// StoredMessage represents a single message in a conversation record.
@@ -76,7 +78,7 @@ func ConvStorePath(tokenFilePath string) string {
}
convDir := filepath.Join(wd, "conv")
base := strings.TrimSuffix(filepath.Base(tokenFilePath), filepath.Ext(tokenFilePath))
return filepath.Join(convDir, base+".conv.json")
return filepath.Join(convDir, base+".bolt")
}
// ConvDataPath returns the path for full conversation persistence based on token file path.
@@ -87,24 +89,41 @@ func ConvDataPath(tokenFilePath string) string {
}
convDir := filepath.Join(wd, "conv")
base := strings.TrimSuffix(filepath.Base(tokenFilePath), filepath.Ext(tokenFilePath))
return filepath.Join(convDir, base+".data.json")
return filepath.Join(convDir, base+".bolt")
}
// LoadConvStore reads the account-level metadata store from disk.
func LoadConvStore(path string) (map[string][]string, error) {
b, err := os.ReadFile(path)
if err != nil {
// Missing file is not an error; return empty map
return map[string][]string{}, nil
}
var tmp map[string][]string
if err := json.Unmarshal(b, &tmp); err != nil {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return nil, err
}
if tmp == nil {
tmp = map[string][]string{}
db, err := bolt.Open(path, 0o600, &bolt.Options{Timeout: time.Second})
if err != nil {
return nil, err
}
return tmp, nil
defer db.Close()
out := map[string][]string{}
err = db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("account_meta"))
if b == nil {
return nil
}
return b.ForEach(func(k, v []byte) error {
var arr []string
if len(v) > 0 {
if e := json.Unmarshal(v, &arr); e != nil {
// Skip malformed entries instead of failing the whole load
return nil
}
}
out[string(k)] = arr
return nil
})
})
if err != nil {
return nil, err
}
return out, nil
}
// SaveConvStore writes the account-level metadata store to disk atomically.
@@ -112,19 +131,36 @@ func SaveConvStore(path string, data map[string][]string) error {
if data == nil {
data = map[string][]string{}
}
payload, err := json.MarshalIndent(data, "", " ")
if err != nil {
return err
}
// Ensure directory exists
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, payload, 0o644); err != nil {
db, err := bolt.Open(path, 0o600, &bolt.Options{Timeout: 2 * time.Second})
if err != nil {
return err
}
return os.Rename(tmp, path)
defer db.Close()
return db.Update(func(tx *bolt.Tx) error {
// Recreate bucket to reflect the given snapshot exactly.
if b := tx.Bucket([]byte("account_meta")); b != nil {
if err := tx.DeleteBucket([]byte("account_meta")); err != nil {
return err
}
}
b, err := tx.CreateBucket([]byte("account_meta"))
if err != nil {
return err
}
for k, v := range data {
enc, e := json.Marshal(v)
if e != nil {
return e
}
if e := b.Put([]byte(k), enc); e != nil {
return e
}
}
return nil
})
}
// AccountMetaKey builds the key for account-level metadata map.
@@ -134,25 +170,48 @@ func AccountMetaKey(email, modelName string) string {
// LoadConvData reads the full conversation data and index from disk.
func LoadConvData(path string) (map[string]ConversationRecord, map[string]string, error) {
b, err := os.ReadFile(path)
if err != nil {
// Missing file is not an error; return empty sets
return map[string]ConversationRecord{}, map[string]string{}, nil
}
var wrapper struct {
Items map[string]ConversationRecord `json:"items"`
Index map[string]string `json:"index"`
}
if err := json.Unmarshal(b, &wrapper); err != nil {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return nil, nil, err
}
if wrapper.Items == nil {
wrapper.Items = map[string]ConversationRecord{}
db, err := bolt.Open(path, 0o600, &bolt.Options{Timeout: time.Second})
if err != nil {
return nil, nil, err
}
if wrapper.Index == nil {
wrapper.Index = map[string]string{}
defer db.Close()
items := map[string]ConversationRecord{}
index := map[string]string{}
err = db.View(func(tx *bolt.Tx) error {
// Load conv_items
if b := tx.Bucket([]byte("conv_items")); b != nil {
if e := b.ForEach(func(k, v []byte) error {
var rec ConversationRecord
if len(v) > 0 {
if e2 := json.Unmarshal(v, &rec); e2 != nil {
// Skip malformed
return nil
}
items[string(k)] = rec
}
return nil
}); e != nil {
return e
}
}
// Load conv_index
if b := tx.Bucket([]byte("conv_index")); b != nil {
if e := b.ForEach(func(k, v []byte) error {
index[string(k)] = string(v)
return nil
}); e != nil {
return e
}
}
return nil
})
if err != nil {
return nil, nil, err
}
return wrapper.Items, wrapper.Index, nil
return items, index, nil
}
// SaveConvData writes the full conversation data and index to disk atomically.
@@ -163,22 +222,52 @@ func SaveConvData(path string, items map[string]ConversationRecord, index map[st
if index == nil {
index = map[string]string{}
}
wrapper := struct {
Items map[string]ConversationRecord `json:"items"`
Index map[string]string `json:"index"`
}{Items: items, Index: index}
payload, err := json.MarshalIndent(wrapper, "", " ")
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, payload, 0o644); err != nil {
db, err := bolt.Open(path, 0o600, &bolt.Options{Timeout: 2 * time.Second})
if err != nil {
return err
}
return os.Rename(tmp, path)
defer db.Close()
return db.Update(func(tx *bolt.Tx) error {
// Recreate items bucket
if b := tx.Bucket([]byte("conv_items")); b != nil {
if err := tx.DeleteBucket([]byte("conv_items")); err != nil {
return err
}
}
bi, err := tx.CreateBucket([]byte("conv_items"))
if err != nil {
return err
}
for k, rec := range items {
enc, e := json.Marshal(rec)
if e != nil {
return e
}
if e := bi.Put([]byte(k), enc); e != nil {
return e
}
}
// Recreate index bucket
if b := tx.Bucket([]byte("conv_index")); b != nil {
if err := tx.DeleteBucket([]byte("conv_index")); err != nil {
return err
}
}
bx, err := tx.CreateBucket([]byte("conv_index"))
if err != nil {
return err
}
for k, v := range index {
if e := bx.Put([]byte(k), []byte(v)); e != nil {
return e
}
}
return nil
})
}
// BuildConversationRecord constructs a ConversationRecord from history and the latest output.