mirror of
https://github.com/yangjian102621/geekai.git
synced 2025-11-10 03:03:43 +08:00
fix: 修复 markdown 换行符不解析的 Bug,修复新发布的模型 token 统计失败错误
This commit is contained in:
@@ -6,7 +6,10 @@ import (
|
||||
)
|
||||
|
||||
func CalcTokens(text string, model string) (int, error) {
|
||||
encoding := tiktoken.MODEL_TO_ENCODING[model]
|
||||
encoding, ok := tiktoken.MODEL_TO_ENCODING[model]
|
||||
if !ok {
|
||||
encoding = "cl100k_base"
|
||||
}
|
||||
tke, err := tiktoken.GetEncoding(encoding)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("getEncoding: %v", err)
|
||||
|
||||
Reference in New Issue
Block a user