Merge commit '3d149fedf45472eff92910324974c762fc37dad6'

relay/adaptor/aiproxy/adaptor.go

@@ -2,14 +2,15 @@ package aiproxy
 
 import (
 	"fmt"
+	"io"
+	"net/http"
+
 	"github.com/Laisky/errors/v2"
-	"github.com/Laisky/one-api/common/config"
+	"github.com/Laisky/one-api/common/ctxkey"
 	"github.com/Laisky/one-api/relay/adaptor"
 	"github.com/Laisky/one-api/relay/meta"
 	"github.com/Laisky/one-api/relay/model"
 	"github.com/gin-gonic/gin"
-	"io"
-	"net/http"
 )
 
 type Adaptor struct {
@@ -34,7 +35,7 @@ func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.G
 		return nil, errors.New("request is nil")
 	}
 	aiProxyLibraryRequest := ConvertRequest(*request)
-	aiProxyLibraryRequest.LibraryId = c.GetString(config.KeyLibraryID)
+	aiProxyLibraryRequest.LibraryId = c.GetString(ctxkey.ConfigLibraryID)
 	return aiProxyLibraryRequest, nil
 }
 

relay/adaptor/ali/adaptor.go

@@ -6,7 +6,7 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/Laisky/one-api/common/config"
+	"github.com/Laisky/one-api/common/ctxkey"
 	"github.com/Laisky/one-api/relay/adaptor"
 	"github.com/Laisky/one-api/relay/meta"
 	"github.com/Laisky/one-api/relay/model"
@@ -48,8 +48,8 @@ func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *me
 	if meta.Mode == relaymode.ImagesGenerations {
 		req.Header.Set("X-DashScope-Async", "enable")
 	}
-	if c.GetString(config.KeyPlugin) != "" {
-		req.Header.Set("X-DashScope-Plugin", c.GetString(config.KeyPlugin))
+	if c.GetString(ctxkey.ConfigPlugin) != "" {
+		req.Header.Set("X-DashScope-Plugin", c.GetString(ctxkey.ConfigPlugin))
 	}
 	return nil
 }

relay/adaptor/aws/main.go

@@ -9,7 +9,6 @@ import (
 	"net/http"
 
 	"github.com/Laisky/one-api/common"
-	"github.com/Laisky/one-api/common/config"
 	"github.com/Laisky/one-api/common/ctxkey"
 	"github.com/Laisky/one-api/common/helper"
 	"github.com/Laisky/one-api/common/logger"
@@ -25,9 +24,9 @@ import (
 )
 
 func newAwsClient(c *gin.Context) (*bedrockruntime.Client, error) {
-	ak := c.GetString(config.KeyAK)
-	sk := c.GetString(config.KeySK)
-	region := c.GetString(config.KeyRegion)
+	ak := c.GetString(ctxkey.ConfigAK)
+	sk := c.GetString(ctxkey.ConfigSK)
+	region := c.GetString(ctxkey.ConfigRegion)
 	client := bedrockruntime.New(bedrockruntime.Options{
 		Region:      region,
 		Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(ak, sk, "")),

relay/adaptor/azure/helper.go

@@ -1,7 +1,7 @@
 package azure
 
 import (
-	"github.com/Laisky/one-api/common/config"
+	"github.com/Laisky/one-api/common/ctxkey"
 	"github.com/gin-gonic/gin"
 )
 
@@ -9,7 +9,7 @@ func GetAPIVersion(c *gin.Context) string {
 	query := c.Request.URL.Query()
 	apiVersion := query.Get("api-version")
 	if apiVersion == "" {
-		apiVersion = c.GetString(config.KeyAPIVersion)
+		apiVersion = c.GetString(ctxkey.ConfigAPIVersion)
 	}
 	return apiVersion
 }

relay/adaptor/coze/adaptor.go

@@ -0,0 +1,76 @@
+package coze
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/Laisky/one-api/common/ctxkey"
+	"github.com/Laisky/one-api/relay/adaptor"
+	"github.com/Laisky/one-api/relay/adaptor/openai"
+	"github.com/Laisky/one-api/relay/meta"
+	"github.com/Laisky/one-api/relay/model"
+	"github.com/gin-gonic/gin"
+)
+
+type Adaptor struct {
+}
+
+func (a *Adaptor) Init(meta *meta.Meta) {
+
+}
+
+func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
+	return fmt.Sprintf("%s/open_api/v2/chat", meta.BaseURL), nil
+}
+
+func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *meta.Meta) error {
+	adaptor.SetupCommonRequestHeader(c, req, meta)
+	req.Header.Set("Authorization", "Bearer "+meta.APIKey)
+	return nil
+}
+
+func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error) {
+	if request == nil {
+		return nil, errors.New("request is nil")
+	}
+	request.User = c.GetString(ctxkey.ConfigUserID)
+	return ConvertRequest(*request), nil
+}
+
+func (a *Adaptor) ConvertImageRequest(request *model.ImageRequest) (any, error) {
+	if request == nil {
+		return nil, errors.New("request is nil")
+	}
+	return request, nil
+}
+
+func (a *Adaptor) DoRequest(c *gin.Context, meta *meta.Meta, requestBody io.Reader) (*http.Response, error) {
+	return adaptor.DoRequestHelper(a, c, meta, requestBody)
+}
+
+func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
+	var responseText *string
+	if meta.IsStream {
+		err, responseText = StreamHandler(c, resp)
+	} else {
+		err, responseText = Handler(c, resp, meta.PromptTokens, meta.ActualModelName)
+	}
+	if responseText != nil {
+		usage = openai.ResponseText2Usage(*responseText, meta.ActualModelName, meta.PromptTokens)
+	} else {
+		usage = &model.Usage{}
+	}
+	usage.PromptTokens = meta.PromptTokens
+	usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens
+	return
+}
+
+func (a *Adaptor) GetModelList() []string {
+	return ModelList
+}
+
+func (a *Adaptor) GetChannelName() string {
+	return "coze"
+}

relay/adaptor/coze/constant/contenttype/define.go

@@ -0,0 +1,5 @@
+package contenttype
+
+const (
+	Text = "text"
+)

relay/adaptor/coze/constant/event/define.go

@@ -0,0 +1,7 @@
+package event
+
+const (
+	Message = "message"
+	Done    = "done"
+	Error   = "error"
+)

relay/adaptor/coze/constant/messagetype/define.go

@@ -0,0 +1,6 @@
+package messagetype
+
+const (
+	Answer   = "answer"
+	FollowUp = "follow_up"
+)

relay/adaptor/coze/constants.go

@@ -0,0 +1,3 @@
+package coze
+
+var ModelList = []string{}

relay/adaptor/coze/helper.go

@@ -0,0 +1,10 @@
+package coze
+
+import "github.com/Laisky/one-api/relay/adaptor/coze/constant/event"
+
+func event2StopReason(e *string) string {
+	if e == nil || *e == event.Message {
+		return ""
+	}
+	return "stop"
+}

relay/adaptor/coze/main.go

@@ -0,0 +1,215 @@
+package coze
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"github.com/Laisky/one-api/common"
+	"github.com/Laisky/one-api/common/conv"
+	"github.com/Laisky/one-api/common/helper"
+	"github.com/Laisky/one-api/common/logger"
+	"github.com/Laisky/one-api/relay/adaptor/coze/constant/messagetype"
+	"github.com/Laisky/one-api/relay/adaptor/openai"
+	"github.com/Laisky/one-api/relay/model"
+	"github.com/gin-gonic/gin"
+	"io"
+	"net/http"
+	"strings"
+)
+
+// https://www.coze.com/open
+
+func stopReasonCoze2OpenAI(reason *string) string {
+	if reason == nil {
+		return ""
+	}
+	switch *reason {
+	case "end_turn":
+		return "stop"
+	case "stop_sequence":
+		return "stop"
+	case "max_tokens":
+		return "length"
+	default:
+		return *reason
+	}
+}
+
+func ConvertRequest(textRequest model.GeneralOpenAIRequest) *Request {
+	cozeRequest := Request{
+		Stream: textRequest.Stream,
+		User:   textRequest.User,
+		BotId:  strings.TrimPrefix(textRequest.Model, "bot-"),
+	}
+	for i, message := range textRequest.Messages {
+		if i == len(textRequest.Messages)-1 {
+			cozeRequest.Query = message.StringContent()
+			continue
+		}
+		cozeMessage := Message{
+			Role:    message.Role,
+			Content: message.StringContent(),
+		}
+		cozeRequest.ChatHistory = append(cozeRequest.ChatHistory, cozeMessage)
+	}
+	return &cozeRequest
+}
+
+func StreamResponseCoze2OpenAI(cozeResponse *StreamResponse) (*openai.ChatCompletionsStreamResponse, *Response) {
+	var response *Response
+	var stopReason string
+	var choice openai.ChatCompletionsStreamResponseChoice
+
+	if cozeResponse.Message != nil {
+		if cozeResponse.Message.Type != messagetype.Answer {
+			return nil, nil
+		}
+		choice.Delta.Content = cozeResponse.Message.Content
+	}
+	choice.Delta.Role = "assistant"
+	finishReason := stopReasonCoze2OpenAI(&stopReason)
+	if finishReason != "null" {
+		choice.FinishReason = &finishReason
+	}
+	var openaiResponse openai.ChatCompletionsStreamResponse
+	openaiResponse.Object = "chat.completion.chunk"
+	openaiResponse.Choices = []openai.ChatCompletionsStreamResponseChoice{choice}
+	openaiResponse.Id = cozeResponse.ConversationId
+	return &openaiResponse, response
+}
+
+func ResponseCoze2OpenAI(cozeResponse *Response) *openai.TextResponse {
+	var responseText string
+	for _, message := range cozeResponse.Messages {
+		if message.Type == messagetype.Answer {
+			responseText = message.Content
+			break
+		}
+	}
+	choice := openai.TextResponseChoice{
+		Index: 0,
+		Message: model.Message{
+			Role:    "assistant",
+			Content: responseText,
+			Name:    nil,
+		},
+		FinishReason: "stop",
+	}
+	fullTextResponse := openai.TextResponse{
+		Id:      fmt.Sprintf("chatcmpl-%s", cozeResponse.ConversationId),
+		Model:   "coze-bot",
+		Object:  "chat.completion",
+		Created: helper.GetTimestamp(),
+		Choices: []openai.TextResponseChoice{choice},
+	}
+	return &fullTextResponse
+}
+
+func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusCode, *string) {
+	var responseText string
+	createdTime := helper.GetTimestamp()
+	scanner := bufio.NewScanner(resp.Body)
+	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
+		if atEOF && len(data) == 0 {
+			return 0, nil, nil
+		}
+		if i := strings.Index(string(data), "\n"); i >= 0 {
+			return i + 1, data[0:i], nil
+		}
+		if atEOF {
+			return len(data), data, nil
+		}
+		return 0, nil, nil
+	})
+	dataChan := make(chan string)
+	stopChan := make(chan bool)
+	go func() {
+		for scanner.Scan() {
+			data := scanner.Text()
+			if len(data) < 5 {
+				continue
+			}
+			if !strings.HasPrefix(data, "data:") {
+				continue
+			}
+			data = strings.TrimPrefix(data, "data:")
+			dataChan <- data
+		}
+		stopChan <- true
+	}()
+	common.SetEventStreamHeaders(c)
+	var modelName string
+	c.Stream(func(w io.Writer) bool {
+		select {
+		case data := <-dataChan:
+			// some implementations may add \r at the end of data
+			data = strings.TrimSuffix(data, "\r")
+			var cozeResponse StreamResponse
+			err := json.Unmarshal([]byte(data), &cozeResponse)
+			if err != nil {
+				logger.SysError("error unmarshalling stream response: " + err.Error())
+				return true
+			}
+			response, _ := StreamResponseCoze2OpenAI(&cozeResponse)
+			if response == nil {
+				return true
+			}
+			for _, choice := range response.Choices {
+				responseText += conv.AsString(choice.Delta.Content)
+			}
+			response.Model = modelName
+			response.Created = createdTime
+			jsonStr, err := json.Marshal(response)
+			if err != nil {
+				logger.SysError("error marshalling stream response: " + err.Error())
+				return true
+			}
+			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonStr)})
+			return true
+		case <-stopChan:
+			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
+			return false
+		}
+	})
+	_ = resp.Body.Close()
+	return nil, &responseText
+}
+
+func Handler(c *gin.Context, resp *http.Response, promptTokens int, modelName string) (*model.ErrorWithStatusCode, *string) {
+	responseBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return openai.ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError), nil
+	}
+	err = resp.Body.Close()
+	if err != nil {
+		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil
+	}
+	var cozeResponse Response
+	err = json.Unmarshal(responseBody, &cozeResponse)
+	if err != nil {
+		return openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError), nil
+	}
+	if cozeResponse.Code != 0 {
+		return &model.ErrorWithStatusCode{
+			Error: model.Error{
+				Message: cozeResponse.Msg,
+				Code:    cozeResponse.Code,
+			},
+			StatusCode: resp.StatusCode,
+		}, nil
+	}
+	fullTextResponse := ResponseCoze2OpenAI(&cozeResponse)
+	fullTextResponse.Model = modelName
+	jsonResponse, err := json.Marshal(fullTextResponse)
+	if err != nil {
+		return openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError), nil
+	}
+	c.Writer.Header().Set("Content-Type", "application/json")
+	c.Writer.WriteHeader(resp.StatusCode)
+	_, err = c.Writer.Write(jsonResponse)
+	var responseText string
+	if len(fullTextResponse.Choices) > 0 {
+		responseText = fullTextResponse.Choices[0].Message.StringContent()
+	}
+	return nil, &responseText
+}

relay/adaptor/coze/model.go

@@ -0,0 +1,38 @@
+package coze
+
+type Message struct {
+	Role        string `json:"role"`
+	Type        string `json:"type"`
+	Content     string `json:"content"`
+	ContentType string `json:"content_type"`
+}
+
+type ErrorInformation struct {
+	Code int    `json:"code"`
+	Msg  string `json:"msg"`
+}
+
+type Request struct {
+	ConversationId string    `json:"conversation_id,omitempty"`
+	BotId          string    `json:"bot_id"`
+	User           string    `json:"user"`
+	Query          string    `json:"query"`
+	ChatHistory    []Message `json:"chat_history,omitempty"`
+	Stream         bool      `json:"stream"`
+}
+
+type Response struct {
+	ConversationId string    `json:"conversation_id,omitempty"`
+	Messages       []Message `json:"messages,omitempty"`
+	Code           int       `json:"code,omitempty"`
+	Msg            string    `json:"msg,omitempty"`
+}
+
+type StreamResponse struct {
+	Event            string            `json:"event,omitempty"`
+	Message          *Message          `json:"message,omitempty"`
+	IsFinish         bool              `json:"is_finish,omitempty"`
+	Index            int               `json:"index,omitempty"`
+	ConversationId   string            `json:"conversation_id,omitempty"`
+	ErrorInformation *ErrorInformation `json:"error_information,omitempty"`
+}

relay/adaptor/groq/constants.go

@@ -7,4 +7,6 @@ var ModelList = []string{
 	"llama2-7b-2048",
 	"llama2-70b-4096",
 	"mixtral-8x7b-32768",
+	"llama3-8b-8192",
+	"llama3-70b-8192",
 }

relay/adaptor/openai/main.go

@@ -16,6 +16,12 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
+const (
+	dataPrefix       = "data: "
+	done             = "[DONE]"
+	dataPrefixLength = len(dataPrefix)
+)
+
 func StreamHandler(c *gin.Context, resp *http.Response, relayMode int) (*model.ErrorWithStatusCode, string, *model.Usage) {
 	responseText := ""
 	scanner := bufio.NewScanner(resp.Body)
@@ -37,39 +43,46 @@ func StreamHandler(c *gin.Context, resp *http.Response, relayMode int) (*model.E
 	go func() {
 		for scanner.Scan() {
 			data := scanner.Text()
-			if len(data) < 6 { // ignore blank line or wrong format
+			if len(data) < dataPrefixLength { // ignore blank line or wrong format
 				continue
 			}
-			if data[:6] != "data: " && data[:6] != "[DONE]" {
+			if data[:dataPrefixLength] != dataPrefix && data[:dataPrefixLength] != done {
 				continue
 			}
-			dataChan <- data
-			data = data[6:]
-			if !strings.HasPrefix(data, "[DONE]") {
-				switch relayMode {
-				case relaymode.ChatCompletions:
-					var streamResponse ChatCompletionsStreamResponse
-					err := json.Unmarshal([]byte(data), &streamResponse)
-					if err != nil {
-						logger.SysError("error unmarshalling stream response: " + err.Error())
-						continue // just ignore the error
-					}
-					for _, choice := range streamResponse.Choices {
-						responseText += conv.AsString(choice.Delta.Content)
-					}
-					if streamResponse.Usage != nil {
-						usage = streamResponse.Usage
-					}
-				case relaymode.Completions:
-					var streamResponse CompletionsStreamResponse
-					err := json.Unmarshal([]byte(data), &streamResponse)
-					if err != nil {
-						logger.SysError("error unmarshalling stream response: " + err.Error())
-						continue
-					}
-					for _, choice := range streamResponse.Choices {
-						responseText += choice.Text
-					}
+			if strings.HasPrefix(data[dataPrefixLength:], done) {
+				dataChan <- data
+				continue
+			}
+			switch relayMode {
+			case relaymode.ChatCompletions:
+				var streamResponse ChatCompletionsStreamResponse
+				err := json.Unmarshal([]byte(data[dataPrefixLength:]), &streamResponse)
+				if err != nil {
+					logger.SysError("error unmarshalling stream response: " + err.Error())
+					dataChan <- data // if error happened, pass the data to client
+					continue         // just ignore the error
+				}
+				if len(streamResponse.Choices) == 0 {
+					// but for empty choice, we should not pass it to client, this is for azure
+					continue // just ignore empty choice
+				}
+				dataChan <- data
+				for _, choice := range streamResponse.Choices {
+					responseText += conv.AsString(choice.Delta.Content)
+				}
+				if streamResponse.Usage != nil {
+					usage = streamResponse.Usage
+				}
+			case relaymode.Completions:
+				dataChan <- data
+				var streamResponse CompletionsStreamResponse
+				err := json.Unmarshal([]byte(data[dataPrefixLength:]), &streamResponse)
+				if err != nil {
+					logger.SysError("error unmarshalling stream response: " + err.Error())
+					continue
+				}
+				for _, choice := range streamResponse.Choices {
+					responseText += choice.Text
 				}
 			}
 		}

relay/adaptor/xunfei/main.go

@@ -13,7 +13,7 @@ import (
 	"time"
 
 	"github.com/Laisky/one-api/common"
-	"github.com/Laisky/one-api/common/config"
+	"github.com/Laisky/one-api/common/ctxkey"
 	"github.com/Laisky/one-api/common/helper"
 	"github.com/Laisky/one-api/common/logger"
 	"github.com/Laisky/one-api/common/random"
@@ -281,7 +281,7 @@ func getAPIVersion(c *gin.Context, modelName string) string {
 		return apiVersion
 
 	}
-	apiVersion = c.GetString(config.KeyAPIVersion)
+	apiVersion = c.GetString(ctxkey.ConfigAPIVersion)
 	if apiVersion != "" {
 		return apiVersion
 	}