Based on QuantumNous/new-api v1.0.0-rc.36 (shallow history squashed). - CNY quota display mode: RMB-denominated top-up and crediting across Epay/Waffo channels - New CheckinButton in both authenticated and public headers (Turnstile supported) - New /subscribe page with public plan snapshot, Alipay/balance payment, auto group upgrade/reset - New /docs 12-section beginner guide with anchor navigation - New home PromoBanner linking to /docs; i18n synced (en/zh/zh-TW/fr/ja/ru/vi) - README: document custom features, live demo https://ai.88531.cn
64 lines
1.6 KiB
Go
64 lines
1.6 KiB
Go
package service
|
|
|
|
import (
|
|
"sync"
|
|
|
|
"github.com/QuantumNous/new-api/common"
|
|
"github.com/tiktoken-go/tokenizer"
|
|
"github.com/tiktoken-go/tokenizer/codec"
|
|
)
|
|
|
|
// tokenEncoderMap won't grow after initialization
|
|
var defaultTokenEncoder tokenizer.Codec
|
|
|
|
// tokenEncoderMap is used to store token encoders for different models
|
|
var tokenEncoderMap = make(map[string]tokenizer.Codec)
|
|
|
|
// tokenEncoderMutex protects tokenEncoderMap for concurrent access
|
|
var tokenEncoderMutex sync.RWMutex
|
|
|
|
func InitTokenEncoders() {
|
|
common.SysLog("initializing token encoders")
|
|
defaultTokenEncoder = codec.NewCl100kBase()
|
|
common.SysLog("token encoders initialized")
|
|
}
|
|
|
|
func getTokenEncoder(model string) tokenizer.Codec {
|
|
// First, try to get the encoder from cache with read lock
|
|
tokenEncoderMutex.RLock()
|
|
if encoder, exists := tokenEncoderMap[model]; exists {
|
|
tokenEncoderMutex.RUnlock()
|
|
return encoder
|
|
}
|
|
tokenEncoderMutex.RUnlock()
|
|
|
|
// If not in cache, create new encoder with write lock
|
|
tokenEncoderMutex.Lock()
|
|
defer tokenEncoderMutex.Unlock()
|
|
|
|
// Double-check if another goroutine already created the encoder
|
|
if encoder, exists := tokenEncoderMap[model]; exists {
|
|
return encoder
|
|
}
|
|
|
|
// Create new encoder
|
|
modelCodec, err := tokenizer.ForModel(tokenizer.Model(model))
|
|
if err != nil {
|
|
// Cache the default encoder for this model to avoid repeated failures
|
|
tokenEncoderMap[model] = defaultTokenEncoder
|
|
return defaultTokenEncoder
|
|
}
|
|
|
|
// Cache the new encoder
|
|
tokenEncoderMap[model] = modelCodec
|
|
return modelCodec
|
|
}
|
|
|
|
func getTokenNum(tokenEncoder tokenizer.Codec, text string) int {
|
|
if text == "" {
|
|
return 0
|
|
}
|
|
tkm, _ := tokenEncoder.Count(text)
|
|
return tkm
|
|
}
|