fix: 修复 markdown 换行符不解析的 Bug,修复新发布的模型 token 统计失败错误

This commit is contained in:
RockYang
2023-06-27 14:18:20 +08:00
parent 3efd5fb77a
commit 92e52a2284
6 changed files with 316 additions and 113 deletions

View File

@@ -6,7 +6,10 @@ import (
)
func CalcTokens(text string, model string) (int, error) {
encoding := tiktoken.MODEL_TO_ENCODING[model]
encoding, ok := tiktoken.MODEL_TO_ENCODING[model]
if !ok {
encoding = "cl100k_base"
}
tke, err := tiktoken.GetEncoding(encoding)
if err != nil {
return 0, fmt.Errorf("getEncoding: %v", err)