Merge remote-tracking branch 'remotes/origin_songquanpeng/main'

# Conflicts:
#	relay/adaptor/openai/adaptor.go
#	relay/controller/text.go

relay/adaptor/openai/adaptor.go

@@ -3,17 +3,19 @@ package openai
 import (
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/relay/adaptor"
 	"github.com/songquanpeng/one-api/relay/adaptor/doubao"
 	"github.com/songquanpeng/one-api/relay/adaptor/minimax"
+	"github.com/songquanpeng/one-api/relay/adaptor/novita"
 	"github.com/songquanpeng/one-api/relay/channeltype"
 	"github.com/songquanpeng/one-api/relay/meta"
 	"github.com/songquanpeng/one-api/relay/model"
 	"github.com/songquanpeng/one-api/relay/relaymode"
-	"io"
-	"net/http"
-	"strings"
 )
 
 type Adaptor struct {
@@ -53,6 +55,8 @@ func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
 			return GetFullRequestURL("https://apivip.aiproxy.io", meta.RequestURLPath, meta.ChannelType), nil
 		}
 		fallthrough
+	case channeltype.Novita:
+		return novita.GetRequestURL(meta)
 	default:
 		return GetFullRequestURL(meta.BaseURL, meta.RequestURLPath, meta.ChannelType), nil
 	}

relay/adaptor/openai/compatible.go

@@ -10,8 +10,10 @@ import (
 	"github.com/songquanpeng/one-api/relay/adaptor/minimax"
 	"github.com/songquanpeng/one-api/relay/adaptor/mistral"
 	"github.com/songquanpeng/one-api/relay/adaptor/moonshot"
+	"github.com/songquanpeng/one-api/relay/adaptor/novita"
 	"github.com/songquanpeng/one-api/relay/adaptor/stepfun"
 	"github.com/songquanpeng/one-api/relay/adaptor/togetherai"
+	"github.com/songquanpeng/one-api/relay/adaptor/siliconflow"
 	"github.com/songquanpeng/one-api/relay/channeltype"
 )
 
@@ -28,6 +30,8 @@ var CompatibleChannels = []int{
 	channeltype.StepFun,
 	channeltype.DeepSeek,
 	channeltype.TogetherAI,
+	channeltype.Novita,
+	channeltype.SiliconFlow,
 }
 
 func GetCompatibleChannelMeta(channelType int) (string, []string) {
@@ -56,6 +60,10 @@ func GetCompatibleChannelMeta(channelType int) (string, []string) {
 		return "together.ai", togetherai.ModelList
 	case channeltype.Doubao:
 		return "doubao", doubao.ModelList
+	case channeltype.Novita:
+		return "novita", novita.ModelList
+	case channeltype.SiliconFlow:
+		return "siliconflow", siliconflow.ModelList
 	default:
 		return "openai", ModelList
 	}

relay/adaptor/openai/constants.go

@@ -8,6 +8,7 @@ var ModelList = []string{
 	"gpt-4-32k", "gpt-4-32k-0314", "gpt-4-32k-0613",
 	"gpt-4-turbo-preview", "gpt-4-turbo", "gpt-4-turbo-2024-04-09",
 	"gpt-4o", "gpt-4o-2024-05-13",
+	"gpt-4o-mini", "gpt-4o-mini-2024-07-18",
 	"gpt-4-vision-preview",
 	"text-embedding-ada-002", "text-embedding-3-small", "text-embedding-3-large",
 	"text-curie-001", "text-babbage-001", "text-ada-001", "text-davinci-002", "text-davinci-003",

relay/adaptor/openai/main.go

@@ -4,15 +4,18 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/songquanpeng/one-api/common/render"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common"
 	"github.com/songquanpeng/one-api/common/conv"
 	"github.com/songquanpeng/one-api/common/logger"
 	"github.com/songquanpeng/one-api/relay/model"
 	"github.com/songquanpeng/one-api/relay/relaymode"
-	"io"
-	"net/http"
-	"strings"
 )
 
 const (
@@ -24,88 +27,72 @@ const (
 func StreamHandler(c *gin.Context, resp *http.Response, relayMode int) (*model.ErrorWithStatusCode, string, *model.Usage) {
 	responseText := ""
 	scanner := bufio.NewScanner(resp.Body)
-	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
-		if atEOF && len(data) == 0 {
-			return 0, nil, nil
-		}
-		if i := strings.Index(string(data), "\n"); i >= 0 {
-			return i + 1, data[0:i], nil
-		}
-		if atEOF {
-			return len(data), data, nil
-		}
-		return 0, nil, nil
-	})
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
+	scanner.Split(bufio.ScanLines)
 	var usage *model.Usage
-	go func() {
-		for scanner.Scan() {
-			data := scanner.Text()
-			if len(data) < dataPrefixLength { // ignore blank line or wrong format
-				continue
-			}
-			if data[:dataPrefixLength] != dataPrefix && data[:dataPrefixLength] != done {
-				continue
-			}
-			if strings.HasPrefix(data[dataPrefixLength:], done) {
-				dataChan <- data
-				continue
-			}
-			switch relayMode {
-			case relaymode.ChatCompletions:
-				var streamResponse ChatCompletionsStreamResponse
-				err := json.Unmarshal([]byte(data[dataPrefixLength:]), &streamResponse)
-				if err != nil {
-					logger.SysError("error unmarshalling stream response: " + err.Error())
-					dataChan <- data // if error happened, pass the data to client
-					continue         // just ignore the error
-				}
-				if len(streamResponse.Choices) == 0 {
-					// but for empty choice, we should not pass it to client, this is for azure
-					continue // just ignore empty choice
-				}
-				dataChan <- data
-				for _, choice := range streamResponse.Choices {
-					responseText += conv.AsString(choice.Delta.Content)
-				}
-				if streamResponse.Usage != nil {
-					usage = streamResponse.Usage
-				}
-			case relaymode.Completions:
-				dataChan <- data
-				var streamResponse CompletionsStreamResponse
-				err := json.Unmarshal([]byte(data[dataPrefixLength:]), &streamResponse)
-				if err != nil {
-					logger.SysError("error unmarshalling stream response: " + err.Error())
-					continue
-				}
-				for _, choice := range streamResponse.Choices {
-					responseText += choice.Text
-				}
-			}
-		}
-		stopChan <- true
-	}()
+
 	common.SetEventStreamHeaders(c)
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			if strings.HasPrefix(data, "data: [DONE]") {
-				data = data[:12]
-			}
-			// some implementations may add \r at the end of data
-			data = strings.TrimSuffix(data, "\r")
-			c.Render(-1, common.CustomEvent{Data: data})
-			return true
-		case <-stopChan:
-			return false
+
+	doneRendered := false
+	for scanner.Scan() {
+		data := scanner.Text()
+		if len(data) < dataPrefixLength { // ignore blank line or wrong format
+			continue
 		}
-	})
+		if data[:dataPrefixLength] != dataPrefix && data[:dataPrefixLength] != done {
+			continue
+		}
+		if strings.HasPrefix(data[dataPrefixLength:], done) {
+			render.StringData(c, data)
+			doneRendered = true
+			continue
+		}
+		switch relayMode {
+		case relaymode.ChatCompletions:
+			var streamResponse ChatCompletionsStreamResponse
+			err := json.Unmarshal([]byte(data[dataPrefixLength:]), &streamResponse)
+			if err != nil {
+				logger.SysError("error unmarshalling stream response: " + err.Error())
+				render.StringData(c, data) // if error happened, pass the data to client
+				continue                   // just ignore the error
+			}
+			if len(streamResponse.Choices) == 0 {
+				// but for empty choice, we should not pass it to client, this is for azure
+				continue // just ignore empty choice
+			}
+			render.StringData(c, data)
+			for _, choice := range streamResponse.Choices {
+				responseText += conv.AsString(choice.Delta.Content)
+			}
+			if streamResponse.Usage != nil {
+				usage = streamResponse.Usage
+			}
+		case relaymode.Completions:
+			render.StringData(c, data)
+			var streamResponse CompletionsStreamResponse
+			err := json.Unmarshal([]byte(data[dataPrefixLength:]), &streamResponse)
+			if err != nil {
+				logger.SysError("error unmarshalling stream response: " + err.Error())
+				continue
+			}
+			for _, choice := range streamResponse.Choices {
+				responseText += choice.Text
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	if !doneRendered {
+		render.Done(c)
+	}
+
 	err := resp.Body.Close()
 	if err != nil {
 		return ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), "", nil
 	}
+
 	return nil, responseText, usage
 }
 
@@ -149,7 +136,7 @@ func Handler(c *gin.Context, resp *http.Response, promptTokens int, modelName st
 		return ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil
 	}
 
-	if textResponse.Usage.TotalTokens == 0 {
+	if textResponse.Usage.TotalTokens == 0 || (textResponse.Usage.PromptTokens == 0 && textResponse.Usage.CompletionTokens == 0) {
 		completionTokens := 0
 		for _, choice := range textResponse.Choices {
 			completionTokens += CountTokenText(choice.Message.StringContent(), modelName)

relay/adaptor/openai/token.go

@@ -97,7 +97,11 @@ func CountTokenMessages(messages []model.Message, model string) int {
 				m := it.(map[string]any)
 				switch m["type"] {
 				case "text":
-					tokenNum += getTokenNum(tokenEncoder, m["text"].(string))
+					if textValue, ok := m["text"]; ok {
+						if textString, ok := textValue.(string); ok {
+							tokenNum += getTokenNum(tokenEncoder, textString)
+						}
+					}
 				case "image_url":
 					imageUrl, ok := m["image_url"].(map[string]any)
 					if ok {
@@ -106,7 +110,7 @@ func CountTokenMessages(messages []model.Message, model string) int {
 						if imageUrl["detail"] != nil {
 							detail = imageUrl["detail"].(string)
 						}
-						imageTokens, err := countImageTokens(url, detail)
+						imageTokens, err := countImageTokens(url, detail, model)
 						if err != nil {
 							logger.SysError("error counting image tokens: " + err.Error())
 						} else {
@@ -130,11 +134,15 @@ const (
 	lowDetailCost         = 85
 	highDetailCostPerTile = 170
 	additionalCost        = 85
+	// gpt-4o-mini cost higher than other model
+	gpt4oMiniLowDetailCost  = 2833
+	gpt4oMiniHighDetailCost = 5667
+	gpt4oMiniAdditionalCost = 2833
 )
 
 // https://platform.openai.com/docs/guides/vision/calculating-costs
 // https://github.com/openai/openai-cookbook/blob/05e3f9be4c7a2ae7ecf029a7c32065b024730ebe/examples/How_to_count_tokens_with_tiktoken.ipynb
-func countImageTokens(url string, detail string) (_ int, err error) {
+func countImageTokens(url string, detail string, model string) (_ int, err error) {
 	var fetchSize = true
 	var width, height int
 	// Reference: https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding
@@ -168,6 +176,9 @@ func countImageTokens(url string, detail string) (_ int, err error) {
 	}
 	switch detail {
 	case "low":
+		if strings.HasPrefix(model, "gpt-4o-mini") {
+			return gpt4oMiniLowDetailCost, nil
+		}
 		return lowDetailCost, nil
 	case "high":
 		if fetchSize {
@@ -187,6 +198,9 @@ func countImageTokens(url string, detail string) (_ int, err error) {
 			height = int(float64(height) * ratio)
 		}
 		numSquares := int(math.Ceil(float64(width)/512) * math.Ceil(float64(height)/512))
+		if strings.HasPrefix(model, "gpt-4o-mini") {
+			return numSquares*gpt4oMiniHighDetailCost + gpt4oMiniAdditionalCost, nil
+		}
 		result := numSquares*highDetailCostPerTile + additionalCost
 		return result, nil
 	default: