Files
CaIon 14a9acd368 feat(custom): CNY native billing, daily check-in button, subscription plans page, usage docs and promo banner
Based on QuantumNous/new-api v1.0.0-rc.36 (shallow history squashed).

- CNY quota display mode: RMB-denominated top-up and crediting across Epay/Waffo channels
- New CheckinButton in both authenticated and public headers (Turnstile supported)
- New /subscribe page with public plan snapshot, Alipay/balance payment, auto group upgrade/reset
- New /docs 12-section beginner guide with anchor navigation
- New home PromoBanner linking to /docs; i18n synced (en/zh/zh-TW/fr/ja/ru/vi)
- README: document custom features, live demo https://ai.88531.cn
2026-09-23 16:28:52 +08:00

64 lines
1.6 KiB
Go

package service
import (
"sync"
"github.com/QuantumNous/new-api/common"
"github.com/tiktoken-go/tokenizer"
"github.com/tiktoken-go/tokenizer/codec"
)
// tokenEncoderMap won't grow after initialization
var defaultTokenEncoder tokenizer.Codec
// tokenEncoderMap is used to store token encoders for different models
var tokenEncoderMap = make(map[string]tokenizer.Codec)
// tokenEncoderMutex protects tokenEncoderMap for concurrent access
var tokenEncoderMutex sync.RWMutex
func InitTokenEncoders() {
common.SysLog("initializing token encoders")
defaultTokenEncoder = codec.NewCl100kBase()
common.SysLog("token encoders initialized")
}
func getTokenEncoder(model string) tokenizer.Codec {
// First, try to get the encoder from cache with read lock
tokenEncoderMutex.RLock()
if encoder, exists := tokenEncoderMap[model]; exists {
tokenEncoderMutex.RUnlock()
return encoder
}
tokenEncoderMutex.RUnlock()
// If not in cache, create new encoder with write lock
tokenEncoderMutex.Lock()
defer tokenEncoderMutex.Unlock()
// Double-check if another goroutine already created the encoder
if encoder, exists := tokenEncoderMap[model]; exists {
return encoder
}
// Create new encoder
modelCodec, err := tokenizer.ForModel(tokenizer.Model(model))
if err != nil {
// Cache the default encoder for this model to avoid repeated failures
tokenEncoderMap[model] = defaultTokenEncoder
return defaultTokenEncoder
}
// Cache the new encoder
tokenEncoderMap[model] = modelCodec
return modelCodec
}
func getTokenNum(tokenEncoder tokenizer.Codec, text string) int {
if text == "" {
return 0
}
tkm, _ := tokenEncoder.Count(text)
return tkm
}