🔖 chore: add gpt-4o tiktoken

common/token.go

@@ -9,12 +9,13 @@ import (
 	"one-api/common/image"
 	"one-api/types"
 
-	"github.com/pkoukk/tiktoken-go"
+	"github.com/MartialBE/tiktoken-go"
 )
 
 var tokenEncoderMap = map[string]*tiktoken.Tiktoken{}
 var gpt35TokenEncoder *tiktoken.Tiktoken
 var gpt4TokenEncoder *tiktoken.Tiktoken
+var gpt4oTokenEncoder *tiktoken.Tiktoken
 
 func InitTokenEncoders() {
 	SysLog("initializing token encoders")
@@ -29,6 +30,11 @@ func InitTokenEncoders() {
 		FatalLog(fmt.Sprintf("failed to get gpt-4 token encoder: %s", err.Error()))
 	}
 
+	gpt4oTokenEncoder, err = tiktoken.EncodingForModel("gpt-4o")
+	if err != nil {
+		FatalLog(fmt.Sprintf("failed to get gpt-4o token encoder: %s", err.Error()))
+	}
+
 	SysLog("token encoders initialized")
 }
 
@@ -40,6 +46,8 @@ func getTokenEncoder(model string) *tiktoken.Tiktoken {
 
 	if strings.HasPrefix(model, "gpt-3.5") {
 		tokenEncoder = gpt35TokenEncoder
+	} else if strings.HasPrefix(model, "gpt-4o") {
+		tokenEncoder = gpt4oTokenEncoder
 	} else if strings.HasPrefix(model, "gpt-4") {
 		tokenEncoder = gpt4TokenEncoder
 	} else {