merge upstream

Signed-off-by: wozulong <>
This commit is contained in:
wozulong
2024-09-10 17:28:04 +08:00
6 changed files with 17 additions and 9 deletions

View File

@@ -40,6 +40,8 @@ func InitTokenEncoders() {
tokenEncoderMap[model] = gpt35TokenEncoder
} else if strings.HasPrefix(model, "gpt-4o") {
tokenEncoderMap[model] = gpt4oTokenEncoder
} else if strings.HasPrefix(model, "chatgpt-4o") {
tokenEncoderMap[model] = gpt4oTokenEncoder
} else if strings.HasPrefix(model, "gpt-4") {
tokenEncoderMap[model] = gpt4TokenEncoder
} else {