merge upstream

Signed-off-by: wozulong <>
This commit is contained in:
wozulong
2024-09-15 17:30:59 +08:00
15 changed files with 151 additions and 49 deletions

View File

@@ -42,6 +42,8 @@ func InitTokenEncoders() {
tokenEncoderMap[model] = gpt4oTokenEncoder
} else if strings.HasPrefix(model, "chatgpt-4o") {
tokenEncoderMap[model] = gpt4oTokenEncoder
} else if "o1" == model || strings.HasPrefix(model, "o1") {
tokenEncoderMap[model] = gpt4oTokenEncoder
} else if strings.HasPrefix(model, "gpt-4") {
tokenEncoderMap[model] = gpt4TokenEncoder
} else {