feat: support minimax's 6.5 models (close #1395)

* fix zhipu embedding error when input is array but not string

* fix: only use the first one

---------

Co-authored-by: 蔡新疆 <cxj@icc.link>
Co-authored-by: JustSong <songquanpeng@foxmail.com>

README.md

@@ -87,6 +87,7 @@ _✨ 通过标准的 OpenAI API 格式访问所有的大模型，开箱即用
    + [x] [DeepSeek](https://www.deepseek.com/)
    + [x] [Cloudflare Workers AI](https://developers.cloudflare.com/workers-ai/)
    + [x] [DeepL](https://www.deepl.com/)
+   + [x] [together.ai](https://www.together.ai/)
 2. 支持配置镜像以及众多[第三方代理服务](https://iamazing.cn/page/openai-api-third-party-services)。
 3. 支持通过**负载均衡**的方式访问多个渠道。
 4. 支持 **stream 模式**，可以通过流式传输实现打字机效果。

controller/channel-test.go

@@ -5,6 +5,15 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
 	"github.com/songquanpeng/one-api/common/config"
 	"github.com/songquanpeng/one-api/common/ctxkey"
 	"github.com/songquanpeng/one-api/common/logger"
@@ -18,14 +27,6 @@ import (
 	"github.com/songquanpeng/one-api/relay/meta"
 	relaymodel "github.com/songquanpeng/one-api/relay/model"
 	"github.com/songquanpeng/one-api/relay/relaymode"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
 
 	"github.com/gin-gonic/gin"
 )
@@ -69,6 +70,7 @@ func testChannel(channel *model.Channel) (err error, openaiErr *relaymodel.Error
 	adaptor.Init(meta)
 	var modelName string
 	modelList := adaptor.GetModelList()
+	modelMap := channel.GetModelMapping()
 	if len(modelList) != 0 {
 		modelName = modelList[0]
 	}
@@ -77,6 +79,9 @@ func testChannel(channel *model.Channel) (err error, openaiErr *relaymodel.Error
 		if len(modelNames) > 0 {
 			modelName = modelNames[0]
 		}
+		if modelMap != nil && modelMap[modelName] != "" {
+			modelName = modelMap[modelName]
+		}
 	}
 	request := buildTestRequest()
 	request.Model = modelName

relay/adaptor/anthropic/main.go

@@ -4,6 +4,10 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common"
 	"github.com/songquanpeng/one-api/common/helper"
@@ -11,9 +15,6 @@ import (
 	"github.com/songquanpeng/one-api/common/logger"
 	"github.com/songquanpeng/one-api/relay/adaptor/openai"
 	"github.com/songquanpeng/one-api/relay/model"
-	"io"
-	"net/http"
-	"strings"
 )
 
 func stopReasonClaude2OpenAI(reason *string) string {
@@ -176,10 +177,10 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 			if len(data) < 6 {
 				continue
 			}
-			if !strings.HasPrefix(data, "data: ") {
+			if !strings.HasPrefix(data, "data:") {
 				continue
 			}
-			data = strings.TrimPrefix(data, "data: ")
+			data = strings.TrimPrefix(data, "data:")
 			dataChan <- data
 		}
 		stopChan <- true
@@ -192,7 +193,7 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 		select {
 		case data := <-dataChan:
 			// some implementations may add \r at the end of data
-			data = strings.TrimSuffix(data, "\r")
+			data = strings.TrimSpace(data)
 			var claudeResponse StreamResponse
 			err := json.Unmarshal([]byte(data), &claudeResponse)
 			if err != nil {

relay/adaptor/gemini/adaptor.go

@@ -3,6 +3,9 @@ package gemini
 import (
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common/config"
 	"github.com/songquanpeng/one-api/common/helper"
@@ -10,8 +13,6 @@ import (
 	"github.com/songquanpeng/one-api/relay/adaptor/openai"
 	"github.com/songquanpeng/one-api/relay/meta"
 	"github.com/songquanpeng/one-api/relay/model"
-	"io"
-	"net/http"
 )
 
 type Adaptor struct {
@@ -25,7 +26,7 @@ func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
 	version := helper.AssignOrDefault(meta.Config.APIVersion, config.GeminiVersion)
 	action := "generateContent"
 	if meta.IsStream {
-		action = "streamGenerateContent"
+		action = "streamGenerateContent?alt=sse"
 	}
 	return fmt.Sprintf("%s/%s/models/%s:%s", meta.BaseURL, version, meta.ActualModelName, action), nil
 }

relay/adaptor/gemini/main.go

@@ -232,8 +232,6 @@ func streamResponseGeminiChat2OpenAI(geminiResponse *ChatResponse) *openai.ChatC
 
 func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusCode, string) {
 	responseText := ""
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
 	scanner := bufio.NewScanner(resp.Body)
 	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
 		if atEOF && len(data) == 0 {
@@ -247,14 +245,16 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 		}
 		return 0, nil, nil
 	})
+	dataChan := make(chan string)
+	stopChan := make(chan bool)
 	go func() {
 		for scanner.Scan() {
 			data := scanner.Text()
 			data = strings.TrimSpace(data)
-			if !strings.HasPrefix(data, "\"text\": \"") {
+			if !strings.HasPrefix(data, "data: ") {
 				continue
 			}
-			data = strings.TrimPrefix(data, "\"text\": \"")
+			data = strings.TrimPrefix(data, "data: ")
 			data = strings.TrimSuffix(data, "\"")
 			dataChan <- data
 		}
@@ -264,23 +264,17 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 	c.Stream(func(w io.Writer) bool {
 		select {
 		case data := <-dataChan:
-			// this is used to prevent annoying \ related format bug
-			data = fmt.Sprintf("{\"content\": \"%s\"}", data)
-			type dummyStruct struct {
-				Content string `json:"content"`
+			var geminiResponse ChatResponse
+			err := json.Unmarshal([]byte(data), &geminiResponse)
+			if err != nil {
+				logger.SysError("error unmarshalling stream response: " + err.Error())
+				return true
 			}
-			var dummy dummyStruct
-			err := json.Unmarshal([]byte(data), &dummy)
-			responseText += dummy.Content
-			var choice openai.ChatCompletionsStreamResponseChoice
-			choice.Delta.Content = dummy.Content
-			response := openai.ChatCompletionsStreamResponse{
-				Id:      fmt.Sprintf("chatcmpl-%s", random.GetUUID()),
-				Object:  "chat.completion.chunk",
-				Created: helper.GetTimestamp(),
-				Model:   "gemini-pro",
-				Choices: []openai.ChatCompletionsStreamResponseChoice{choice},
+			response := streamResponseGeminiChat2OpenAI(&geminiResponse)
+			if response == nil {
+				return true
 			}
+			responseText += response.Choices[0].Delta.StringContent()
 			jsonResponse, err := json.Marshal(response)
 			if err != nil {
 				logger.SysError("error marshalling stream response: " + err.Error())

relay/adaptor/minimax/constants.go

@@ -1,7 +1,11 @@
 package minimax
 
+// https://www.minimaxi.com/document/guides/chat-model/V2?id=65e0736ab2845de20908e2dd
+
 var ModelList = []string{
-	"abab5.5s-chat",
-	"abab5.5-chat",
+	"abab6.5-chat",
+	"abab6.5s-chat",
 	"abab6-chat",
+	"abab5.5-chat",
+	"abab5.5s-chat",
 }

relay/adaptor/ollama/main.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common"
+	"github.com/songquanpeng/one-api/common/image"
 	"github.com/songquanpeng/one-api/common/logger"
 	"github.com/songquanpeng/one-api/relay/adaptor/openai"
 	"github.com/songquanpeng/one-api/relay/constant"
@@ -32,9 +33,22 @@ func ConvertRequest(request model.GeneralOpenAIRequest) *ChatRequest {
 		Stream: request.Stream,
 	}
 	for _, message := range request.Messages {
+		openaiContent := message.ParseContent()
+		var imageUrls []string
+		var contentText string
+		for _, part := range openaiContent {
+			switch part.Type {
+			case model.ContentTypeText:
+				contentText = part.Text
+			case model.ContentTypeImageURL:
+				_, data, _ := image.GetImageFromUrl(part.ImageURL.Url)
+				imageUrls = append(imageUrls, data)
+			}
+		}
 		ollamaRequest.Messages = append(ollamaRequest.Messages, Message{
 			Role:    message.Role,
-			Content: message.StringContent(),
+			Content: contentText,
+			Images:  imageUrls,
 		})
 	}
 	return &ollamaRequest

relay/adaptor/openai/adaptor.go

@@ -86,9 +86,13 @@ func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *meta.Met
 	if meta.IsStream {
 		var responseText string
 		err, responseText, usage = StreamHandler(c, resp, meta.Mode)
-		if usage == nil {
+		if usage == nil || usage.TotalTokens == 0 {
 			usage = ResponseText2Usage(responseText, meta.ActualModelName, meta.PromptTokens)
 		}
+		if usage.TotalTokens != 0 && usage.PromptTokens == 0 { // some channels don't return prompt tokens & completion tokens
+			usage.PromptTokens = meta.PromptTokens
+			usage.CompletionTokens = usage.TotalTokens - meta.PromptTokens
+		}
 	} else {
 		switch meta.Mode {
 		case relaymode.ImagesGenerations:

relay/adaptor/openai/compatible.go

@@ -10,6 +10,7 @@ import (
 	"github.com/songquanpeng/one-api/relay/adaptor/mistral"
 	"github.com/songquanpeng/one-api/relay/adaptor/moonshot"
 	"github.com/songquanpeng/one-api/relay/adaptor/stepfun"
+	"github.com/songquanpeng/one-api/relay/adaptor/togetherai"
 	"github.com/songquanpeng/one-api/relay/channeltype"
 )
 
@@ -24,6 +25,7 @@ var CompatibleChannels = []int{
 	channeltype.LingYiWanWu,
 	channeltype.StepFun,
 	channeltype.DeepSeek,
+	channeltype.TogetherAI,
 }
 
 func GetCompatibleChannelMeta(channelType int) (string, []string) {
@@ -48,6 +50,8 @@ func GetCompatibleChannelMeta(channelType int) (string, []string) {
 		return "stepfun", stepfun.ModelList
 	case channeltype.DeepSeek:
 		return "deepseek", deepseek.ModelList
+	case channeltype.TogetherAI:
+		return "together.ai", togetherai.ModelList
 	default:
 		return "openai", ModelList
 	}

relay/adaptor/togetherai/constants.go

@@ -0,0 +1,10 @@
+package togetherai
+
+// https://docs.together.ai/docs/inference-models
+
+var ModelList = []string{
+	"meta-llama/Llama-3-70b-chat-hf",
+	"deepseek-ai/deepseek-coder-33b-instruct",
+	"mistralai/Mixtral-8x22B-Instruct-v0.1",
+	"Qwen/Qwen1.5-72B-Chat",
+}

relay/adaptor/zhipu/adaptor.go

@@ -62,8 +62,8 @@ func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.G
 	}
 	switch relayMode {
 	case relaymode.Embeddings:
-		baiduEmbeddingRequest := ConvertEmbeddingRequest(*request)
-		return baiduEmbeddingRequest, nil
+		baiduEmbeddingRequest, err := ConvertEmbeddingRequest(*request)
+		return baiduEmbeddingRequest, err
 	default:
 		// TopP (0.0, 1.0)
 		request.TopP = math.Min(0.99, request.TopP)
@@ -129,11 +129,15 @@ func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *meta.Met
 	return
 }
 
-func ConvertEmbeddingRequest(request model.GeneralOpenAIRequest) *EmbeddingRequest {
-	return &EmbeddingRequest{
-		Model: "embedding-2",
-		Input: request.Input.(string),
+func ConvertEmbeddingRequest(request model.GeneralOpenAIRequest) (*EmbeddingRequest, error) {
+	inputs := request.ParseInput()
+	if len(inputs) != 1 {
+		return nil, errors.New("invalid input length, zhipu only support one input")
 	}
+	return &EmbeddingRequest{
+		Model: request.Model,
+		Input: inputs[0],
+	}, nil
 }
 
 func (a *Adaptor) GetModelList() []string {

relay/billing/ratio/model.go

@@ -138,6 +138,8 @@ var ModelRatio = map[string]float64{
 	"Baichuan2-Turbo-192k": 0.016 * RMB,
 	"Baichuan2-53B":        0.02 * RMB,
 	// https://api.minimax.chat/document/price
+	"abab6.5-chat":  0.03 * RMB,
+	"abab6.5s-chat": 0.01 * RMB,
 	"abab6-chat":    0.1 * RMB,
 	"abab5.5-chat":  0.015 * RMB,
 	"abab5.5s-chat": 0.005 * RMB,

relay/channeltype/define.go

@@ -40,6 +40,7 @@ const (
 	DeepSeek
 	Cloudflare
 	DeepL
+	TogetherAI
 
 	Dummy
 )

relay/channeltype/url.go

@@ -40,6 +40,7 @@ var ChannelBaseURLs = []string{
 	"https://api.deepseek.com",                  // 36
 	"https://api.cloudflare.com",                // 37
 	"https://api-free.deepl.com",                // 38
+	"https://api.together.xyz",                  // 39
 }
 
 func init() {

relay/controller/error.go

@@ -53,6 +53,16 @@ func (e GeneralErrorResponse) ToMessage() string {
 }
 
 func RelayErrorHandler(resp *http.Response) (ErrorWithStatusCode *model.ErrorWithStatusCode) {
+	if resp == nil {
+		return &model.ErrorWithStatusCode{
+			StatusCode: 500,
+			Error: model.Error{
+				Message: "resp is nil",
+				Type:    "upstream_error",
+				Code:    "bad_response",
+			},
+		}
+	}
 	ErrorWithStatusCode = &model.ErrorWithStatusCode{
 		StatusCode: resp.StatusCode,
 		Error: model.Error{

relay/controller/helper.go

@@ -125,9 +125,9 @@ func getPromptTokens(textRequest *relaymodel.GeneralOpenAIRequest, relayMode int
 }
 
 func getPreConsumedQuota(textRequest *relaymodel.GeneralOpenAIRequest, promptTokens int, ratio float64) int64 {
-	preConsumedTokens := config.PreConsumedQuota
+	preConsumedTokens := config.PreConsumedQuota + int64(promptTokens)
 	if textRequest.MaxTokens != 0 {
-		preConsumedTokens = int64(promptTokens) + int64(textRequest.MaxTokens)
+		preConsumedTokens += int64(textRequest.MaxTokens)
 	}
 	return int64(float64(preConsumedTokens) * ratio)
 }
@@ -208,6 +208,9 @@ func getMappedModelName(modelName string, mapping map[string]string) (string, bo
 
 func isErrorHappened(meta *meta.Meta, resp *http.Response) bool {
 	if resp == nil {
+		if meta.ChannelType == channeltype.AwsClaude {
+			return false
+		}
 		return true
 	}
 	if resp.StatusCode != http.StatusOK {

web/berry/src/constants/ChannelConstants.js

@@ -143,6 +143,12 @@ export const CHANNEL_OPTIONS = {
     value: 38,
     color: 'primary'
   },
+  39: {
+    key: 39,
+    text: 'together.ai',
+    value: 39,
+    color: 'primary'
+  },
   8: {
     key: 8,
     text: '自定义渠道',

web/default/src/constants/channel.constants.js

@@ -24,6 +24,7 @@ export const CHANNEL_OPTIONS = [
     {key: 36, text: 'DeepSeek', value: 36, color: 'black'},
     {key: 37, text: 'Cloudflare', value: 37, color: 'orange'},
     {key: 38, text: 'DeepL', value: 38, color: 'black'},
+    {key: 39, text: 'together.ai', value: 39, color: 'blue'},
     {key: 8, text: '自定义渠道', value: 8, color: 'pink'},
     {key: 22, text: '知识库：FastGPT', value: 22, color: 'blue'},
     {key: 21, text: '知识库：AI Proxy', value: 21, color: 'purple'},