mirror of
https://github.com/yangjian102621/geekai.git
synced 2025-09-18 01:06:39 +08:00
18 lines
347 B
Go
18 lines
347 B
Go
package utils
|
|
|
|
import (
|
|
"fmt"
|
|
"github.com/pkoukk/tiktoken-go"
|
|
)
|
|
|
|
func CalcTokens(text string, model string) (int, error) {
|
|
encoding := tiktoken.MODEL_TO_ENCODING[model]
|
|
tke, err := tiktoken.GetEncoding(encoding)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("getEncoding: %v", err)
|
|
}
|
|
|
|
token := tke.Encode(text, nil, nil)
|
|
return len(token), nil
|
|
}
|