From cae8f6a948a2477b1a4ec6a651d3402a3d9af72a Mon Sep 17 00:00:00 2001 From: ybyang Date: Mon, 25 Nov 2024 11:40:10 +0800 Subject: [PATCH 01/10] support maas for xunfei --- relay/adaptor/xunfei/main.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/relay/adaptor/xunfei/main.go b/relay/adaptor/xunfei/main.go index 3984ba5a..0e8155bb 100644 --- a/relay/adaptor/xunfei/main.go +++ b/relay/adaptor/xunfei/main.go @@ -153,7 +153,7 @@ func buildXunfeiAuthUrl(hostUrl string, apiKey, apiSecret string) string { } func StreamHandler(c *gin.Context, meta *meta.Meta, textRequest model.GeneralOpenAIRequest, appId string, apiSecret string, apiKey string) (*model.ErrorWithStatusCode, *model.Usage) { - domain, authUrl := getXunfeiAuthUrl(meta.Config.APIVersion, apiKey, apiSecret) + domain, authUrl := getXunfeiAuthUrl(meta.Config.APIVersion, apiKey, apiSecret, textRequest.Model) dataChan, stopChan, err := xunfeiMakeRequest(textRequest, domain, authUrl, appId) if err != nil { return openai.ErrorWrapper(err, "xunfei_request_failed", http.StatusInternalServerError), nil @@ -183,7 +183,7 @@ func StreamHandler(c *gin.Context, meta *meta.Meta, textRequest model.GeneralOpe } func Handler(c *gin.Context, meta *meta.Meta, textRequest model.GeneralOpenAIRequest, appId string, apiSecret string, apiKey string) (*model.ErrorWithStatusCode, *model.Usage) { - domain, authUrl := getXunfeiAuthUrl(meta.Config.APIVersion, apiKey, apiSecret) + domain, authUrl := getXunfeiAuthUrl(meta.Config.APIVersion, apiKey, apiSecret, textRequest.Model) dataChan, stopChan, err := xunfeiMakeRequest(textRequest, domain, authUrl, appId) if err != nil { return openai.ErrorWrapper(err, "xunfei_request_failed", http.StatusInternalServerError), nil @@ -300,7 +300,7 @@ func apiVersion2domain(apiVersion string) string { return "general" + apiVersion } -func getXunfeiAuthUrl(apiVersion string, apiKey string, apiSecret string) (string, string) { +func getXunfeiAuthUrl(apiVersion string, apiKey string, apiSecret string, modelName string) (string, string) { var authUrl string domain := apiVersion2domain(apiVersion) switch apiVersion { @@ -310,6 +310,13 @@ func getXunfeiAuthUrl(apiVersion string, apiKey string, apiSecret string) (strin case "v3.5-32K": authUrl = buildXunfeiAuthUrl(fmt.Sprintf("wss://spark-api.xf-yun.com/chat/max-32k"), apiKey, apiSecret) break + case "maas": + domain = modelName + authUrl = buildXunfeiAuthUrl(fmt.Sprintf("wss://maas-api.cn-huabei-1.xf-yun.com/%s/chat", apiVersion), apiKey, apiSecret) + case "xingchen": + domain = modelName + authUrl = buildXunfeiAuthUrl(fmt.Sprintf("wss://xingcheng-api.cn-huabei-1.xf-yun.com/%s/chat", apiVersion), apiKey, apiSecret) + default: authUrl = buildXunfeiAuthUrl(fmt.Sprintf("wss://spark-api.xf-yun.com/%s/chat", apiVersion), apiKey, apiSecret) } From 62ab7f47adb7336cdce019226eb89c20a5c7e1bc Mon Sep 17 00:00:00 2001 From: ybyang Date: Mon, 25 Nov 2024 13:36:43 +0800 Subject: [PATCH 02/10] support maas for xunfei --- relay/adaptor/xunfei/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relay/adaptor/xunfei/main.go b/relay/adaptor/xunfei/main.go index 0e8155bb..842f60da 100644 --- a/relay/adaptor/xunfei/main.go +++ b/relay/adaptor/xunfei/main.go @@ -312,10 +312,10 @@ func getXunfeiAuthUrl(apiVersion string, apiKey string, apiSecret string, modelN break case "maas": domain = modelName - authUrl = buildXunfeiAuthUrl(fmt.Sprintf("wss://maas-api.cn-huabei-1.xf-yun.com/%s/chat", apiVersion), apiKey, apiSecret) + authUrl = buildXunfeiAuthUrl(fmt.Sprintf("wss://maas-api.cn-huabei-1.xf-yun.com/v1.1/chat"), apiKey, apiSecret) case "xingchen": domain = modelName - authUrl = buildXunfeiAuthUrl(fmt.Sprintf("wss://xingcheng-api.cn-huabei-1.xf-yun.com/%s/chat", apiVersion), apiKey, apiSecret) + authUrl = buildXunfeiAuthUrl(fmt.Sprintf("wss://xingcheng-api.cn-huabei-1.xf-yun.com/v1.1/chat"), apiKey, apiSecret) default: authUrl = buildXunfeiAuthUrl(fmt.Sprintf("wss://spark-api.xf-yun.com/%s/chat", apiVersion), apiKey, apiSecret) From ff24d0fab87c8adfb4cee8e26a5d92b0dbf006c4 Mon Sep 17 00:00:00 2001 From: ybyang Date: Mon, 25 Nov 2024 14:27:03 +0800 Subject: [PATCH 03/10] =?UTF-8?q?=E6=94=AF=E6=8C=81LoraID=20=EF=BC=8C=20?= =?UTF-8?q?=E9=80=9A=E8=BF=87OpenAI=20Client=E8=AE=BE=E7=BD=AE=20ExtraHead?= =?UTF-8?q?er=20=E5=B9=B6=E5=9C=A8=E8=AE=AF=E9=A3=9E=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E4=B8=AD=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/ctxkey/key.go | 1 + relay/adaptor/xunfei/main.go | 11 ++++++----- relay/adaptor/xunfei/model.go | 3 ++- relay/meta/relay_meta.go | 4 ++++ 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/common/ctxkey/key.go b/common/ctxkey/key.go index 115558a5..edfccf4a 100644 --- a/common/ctxkey/key.go +++ b/common/ctxkey/key.go @@ -21,4 +21,5 @@ const ( AvailableModels = "available_models" KeyRequestBody = "key_request_body" SystemPrompt = "system_prompt" + LoraId = "lora_id" ) diff --git a/relay/adaptor/xunfei/main.go b/relay/adaptor/xunfei/main.go index 842f60da..516bea5b 100644 --- a/relay/adaptor/xunfei/main.go +++ b/relay/adaptor/xunfei/main.go @@ -28,7 +28,7 @@ import ( // https://console.xfyun.cn/services/cbm // https://www.xfyun.cn/doc/spark/Web.html -func requestOpenAI2Xunfei(request model.GeneralOpenAIRequest, xunfeiAppId string, domain string) *ChatRequest { +func requestOpenAI2Xunfei(request model.GeneralOpenAIRequest, xunfeiAppId string, domain string, xunfeiPatchId string) *ChatRequest { messages := make([]Message, 0, len(request.Messages)) for _, message := range request.Messages { messages = append(messages, Message{ @@ -38,6 +38,7 @@ func requestOpenAI2Xunfei(request model.GeneralOpenAIRequest, xunfeiAppId string } xunfeiRequest := ChatRequest{} xunfeiRequest.Header.AppId = xunfeiAppId + xunfeiRequest.Header.PatchId = xunfeiPatchId xunfeiRequest.Parameter.Chat.Domain = domain xunfeiRequest.Parameter.Chat.Temperature = request.Temperature xunfeiRequest.Parameter.Chat.TopK = request.N @@ -154,7 +155,7 @@ func buildXunfeiAuthUrl(hostUrl string, apiKey, apiSecret string) string { func StreamHandler(c *gin.Context, meta *meta.Meta, textRequest model.GeneralOpenAIRequest, appId string, apiSecret string, apiKey string) (*model.ErrorWithStatusCode, *model.Usage) { domain, authUrl := getXunfeiAuthUrl(meta.Config.APIVersion, apiKey, apiSecret, textRequest.Model) - dataChan, stopChan, err := xunfeiMakeRequest(textRequest, domain, authUrl, appId) + dataChan, stopChan, err := xunfeiMakeRequest(textRequest, domain, authUrl, appId, meta.LoraId) if err != nil { return openai.ErrorWrapper(err, "xunfei_request_failed", http.StatusInternalServerError), nil } @@ -184,7 +185,7 @@ func StreamHandler(c *gin.Context, meta *meta.Meta, textRequest model.GeneralOpe func Handler(c *gin.Context, meta *meta.Meta, textRequest model.GeneralOpenAIRequest, appId string, apiSecret string, apiKey string) (*model.ErrorWithStatusCode, *model.Usage) { domain, authUrl := getXunfeiAuthUrl(meta.Config.APIVersion, apiKey, apiSecret, textRequest.Model) - dataChan, stopChan, err := xunfeiMakeRequest(textRequest, domain, authUrl, appId) + dataChan, stopChan, err := xunfeiMakeRequest(textRequest, domain, authUrl, appId, meta.LoraId) if err != nil { return openai.ErrorWrapper(err, "xunfei_request_failed", http.StatusInternalServerError), nil } @@ -220,7 +221,7 @@ func Handler(c *gin.Context, meta *meta.Meta, textRequest model.GeneralOpenAIReq return nil, &usage } -func xunfeiMakeRequest(textRequest model.GeneralOpenAIRequest, domain, authUrl, appId string) (chan ChatResponse, chan bool, error) { +func xunfeiMakeRequest(textRequest model.GeneralOpenAIRequest, domain, authUrl, appId, patchId string) (chan ChatResponse, chan bool, error) { d := websocket.Dialer{ HandshakeTimeout: 5 * time.Second, } @@ -228,7 +229,7 @@ func xunfeiMakeRequest(textRequest model.GeneralOpenAIRequest, domain, authUrl, if err != nil || resp.StatusCode != 101 { return nil, nil, err } - data := requestOpenAI2Xunfei(textRequest, appId, domain) + data := requestOpenAI2Xunfei(textRequest, appId, domain, patchId) err = conn.WriteJSON(data) if err != nil { return nil, nil, err diff --git a/relay/adaptor/xunfei/model.go b/relay/adaptor/xunfei/model.go index c9fb1bb8..3741a6e5 100644 --- a/relay/adaptor/xunfei/model.go +++ b/relay/adaptor/xunfei/model.go @@ -15,7 +15,8 @@ type Functions struct { type ChatRequest struct { Header struct { - AppId string `json:"app_id"` + AppId string `json:"app_id"` + PatchId string `json:"patch_id,omitempty"` } `json:"header"` Parameter struct { Chat struct { diff --git a/relay/meta/relay_meta.go b/relay/meta/relay_meta.go index bcbe1045..461c244c 100644 --- a/relay/meta/relay_meta.go +++ b/relay/meta/relay_meta.go @@ -31,6 +31,9 @@ type Meta struct { RequestURLPath string PromptTokens int // only for DoResponse SystemPrompt string + + //Lora_id + LoraId string } func GetByContext(c *gin.Context) *Meta { @@ -48,6 +51,7 @@ func GetByContext(c *gin.Context) *Meta { APIKey: strings.TrimPrefix(c.Request.Header.Get("Authorization"), "Bearer "), RequestURLPath: c.Request.URL.String(), SystemPrompt: c.GetString(ctxkey.SystemPrompt), + LoraId: c.Request.Header.Get(ctxkey.LoraId), } cfg, ok := c.Get(ctxkey.Config) if ok { From e7cc50d8cb971194d7db733faab92822f6ca9015 Mon Sep 17 00:00:00 2001 From: ybyang Date: Mon, 25 Nov 2024 15:49:52 +0800 Subject: [PATCH 04/10] =?UTF-8?q?=E8=AE=AF=E9=A3=9E=E5=93=8D=E5=BA=94=20si?= =?UTF-8?q?d=20=E9=94=99=E8=AF=AF=E7=A0=81=E6=8F=90=E7=A4=BA=E8=BF=94?= =?UTF-8?q?=E5=9B=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- relay/adaptor/xunfei/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/relay/adaptor/xunfei/main.go b/relay/adaptor/xunfei/main.go index 516bea5b..3b171b1f 100644 --- a/relay/adaptor/xunfei/main.go +++ b/relay/adaptor/xunfei/main.go @@ -10,6 +10,7 @@ import ( "io" "net/http" "net/url" + "strconv" "strings" "time" @@ -206,6 +207,10 @@ func Handler(c *gin.Context, meta *meta.Meta, textRequest model.GeneralOpenAIReq case stop = <-stopChan: } } + if xunfeiResponse.Header.Code != 0 { + return openai.ErrorWrapper(errors.New("xunfei response error: sid: "+xunfeiResponse.Header.Sid), strconv.Itoa(xunfeiResponse.Header.Code), http.StatusInternalServerError), nil + + } if len(xunfeiResponse.Payload.Choices.Text) == 0 { return openai.ErrorWrapper(errors.New("xunfei empty response detected"), "xunfei_empty_response_detected", http.StatusInternalServerError), nil } From 4c395352ff1122710477dc0ab7510ae4239b5740 Mon Sep 17 00:00:00 2001 From: ybyang Date: Mon, 25 Nov 2024 16:58:08 +0800 Subject: [PATCH 05/10] =?UTF-8?q?=E8=AE=AF=E9=A3=9E=E5=93=8D=E5=BA=94=20si?= =?UTF-8?q?d=20=E9=94=99=E8=AF=AF=E7=A0=81=E6=8F=90=E7=A4=BA=E8=BF=94?= =?UTF-8?q?=E5=9B=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- relay/adaptor/xunfei/main.go | 4 +++- relay/adaptor/xunfei/model.go | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/relay/adaptor/xunfei/main.go b/relay/adaptor/xunfei/main.go index 3b171b1f..cadc4c48 100644 --- a/relay/adaptor/xunfei/main.go +++ b/relay/adaptor/xunfei/main.go @@ -39,7 +39,9 @@ func requestOpenAI2Xunfei(request model.GeneralOpenAIRequest, xunfeiAppId string } xunfeiRequest := ChatRequest{} xunfeiRequest.Header.AppId = xunfeiAppId - xunfeiRequest.Header.PatchId = xunfeiPatchId + if xunfeiPatchId != "" { + xunfeiRequest.Header.PatchId = []string{xunfeiPatchId} + } xunfeiRequest.Parameter.Chat.Domain = domain xunfeiRequest.Parameter.Chat.Temperature = request.Temperature xunfeiRequest.Parameter.Chat.TopK = request.N diff --git a/relay/adaptor/xunfei/model.go b/relay/adaptor/xunfei/model.go index 3741a6e5..e22b22e4 100644 --- a/relay/adaptor/xunfei/model.go +++ b/relay/adaptor/xunfei/model.go @@ -15,8 +15,8 @@ type Functions struct { type ChatRequest struct { Header struct { - AppId string `json:"app_id"` - PatchId string `json:"patch_id,omitempty"` + AppId string `json:"app_id"` + PatchId []string `json:"patch_id,omitempty"` } `json:"header"` Parameter struct { Chat struct { From e916af9c566aa6d8250e4e1bc15dc99a3dad7e1d Mon Sep 17 00:00:00 2001 From: ybyang Date: Mon, 25 Nov 2024 17:05:33 +0800 Subject: [PATCH 06/10] =?UTF-8?q?=E8=AE=AF=E9=A3=9E=E5=93=8D=E5=BA=94=20?= =?UTF-8?q?=E8=AE=BE=E7=BD=AEsid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- relay/adaptor/xunfei/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/adaptor/xunfei/main.go b/relay/adaptor/xunfei/main.go index cadc4c48..6f100e8b 100644 --- a/relay/adaptor/xunfei/main.go +++ b/relay/adaptor/xunfei/main.go @@ -97,7 +97,7 @@ func responseXunfei2OpenAI(response *ChatResponse) *openai.TextResponse { FinishReason: constant.StopFinishReason, } fullTextResponse := openai.TextResponse{ - Id: fmt.Sprintf("chatcmpl-%s", random.GetUUID()), + Id: response.Header.Sid, Object: "chat.completion", Created: helper.GetTimestamp(), Choices: []openai.TextResponseChoice{choice}, From 59a835ede7d9325a3af55e26c47725b0c849bfe8 Mon Sep 17 00:00:00 2001 From: ybyang Date: Thu, 28 Nov 2024 18:05:38 +0800 Subject: [PATCH 07/10] support stream usage --- relay/adaptor/xunfei/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/relay/adaptor/xunfei/main.go b/relay/adaptor/xunfei/main.go index 6f100e8b..359cd49a 100644 --- a/relay/adaptor/xunfei/main.go +++ b/relay/adaptor/xunfei/main.go @@ -121,11 +121,12 @@ func streamResponseXunfei2OpenAI(xunfeiResponse *ChatResponse) *openai.ChatCompl choice.FinishReason = &constant.StopFinishReason } response := openai.ChatCompletionsStreamResponse{ - Id: fmt.Sprintf("chatcmpl-%s", random.GetUUID()), + Id: xunfeiResponse.Header.Sid, Object: "chat.completion.chunk", Created: helper.GetTimestamp(), Model: "SparkDesk", Choices: []openai.ChatCompletionsStreamResponseChoice{choice}, + Usage: &xunfeiResponse.Payload.Usage.Text, } return &response } From bb22dc066155f38a7f0aece5c6419c77adec2434 Mon Sep 17 00:00:00 2001 From: ybyang Date: Thu, 28 Nov 2024 18:14:45 +0800 Subject: [PATCH 08/10] support stream usage --- relay/adaptor/xunfei/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/relay/adaptor/xunfei/main.go b/relay/adaptor/xunfei/main.go index 359cd49a..c66ddf06 100644 --- a/relay/adaptor/xunfei/main.go +++ b/relay/adaptor/xunfei/main.go @@ -106,7 +106,7 @@ func responseXunfei2OpenAI(response *ChatResponse) *openai.TextResponse { return &fullTextResponse } -func streamResponseXunfei2OpenAI(xunfeiResponse *ChatResponse) *openai.ChatCompletionsStreamResponse { +func streamResponseXunfei2OpenAI(xunfeiResponse *ChatResponse, usage *model.Usage) *openai.ChatCompletionsStreamResponse { if len(xunfeiResponse.Payload.Choices.Text) == 0 { xunfeiResponse.Payload.Choices.Text = []ChatResponseTextItem{ { @@ -126,7 +126,7 @@ func streamResponseXunfei2OpenAI(xunfeiResponse *ChatResponse) *openai.ChatCompl Created: helper.GetTimestamp(), Model: "SparkDesk", Choices: []openai.ChatCompletionsStreamResponseChoice{choice}, - Usage: &xunfeiResponse.Payload.Usage.Text, + Usage: usage, } return &response } @@ -171,7 +171,7 @@ func StreamHandler(c *gin.Context, meta *meta.Meta, textRequest model.GeneralOpe usage.PromptTokens += xunfeiResponse.Payload.Usage.Text.PromptTokens usage.CompletionTokens += xunfeiResponse.Payload.Usage.Text.CompletionTokens usage.TotalTokens += xunfeiResponse.Payload.Usage.Text.TotalTokens - response := streamResponseXunfei2OpenAI(&xunfeiResponse) + response := streamResponseXunfei2OpenAI(&xunfeiResponse, &usage) jsonResponse, err := json.Marshal(response) if err != nil { logger.SysError("error marshalling stream response: " + err.Error()) From 49d4089d625551e49f165ebfa059b6e480eabf36 Mon Sep 17 00:00:00 2001 From: ybyang Date: Thu, 28 Nov 2024 18:05:38 +0800 Subject: [PATCH 09/10] feat: support stream usage for xunfei stream adaptor From 924d0d8487b3bb46cf80a8481a3f584fbc6e3d3a Mon Sep 17 00:00:00 2001 From: ybyang Date: Sun, 1 Dec 2024 05:34:44 +0800 Subject: [PATCH 10/10] stream error reuturn --- relay/adaptor/xunfei/main.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/relay/adaptor/xunfei/main.go b/relay/adaptor/xunfei/main.go index 6f100e8b..164799dc 100644 --- a/relay/adaptor/xunfei/main.go +++ b/relay/adaptor/xunfei/main.go @@ -106,7 +106,7 @@ func responseXunfei2OpenAI(response *ChatResponse) *openai.TextResponse { return &fullTextResponse } -func streamResponseXunfei2OpenAI(xunfeiResponse *ChatResponse) *openai.ChatCompletionsStreamResponse { +func streamResponseXunfei2OpenAI(xunfeiResponse *ChatResponse, usage *model.Usage) *openai.ChatCompletionsStreamResponse { if len(xunfeiResponse.Payload.Choices.Text) == 0 { xunfeiResponse.Payload.Choices.Text = []ChatResponseTextItem{ { @@ -126,6 +126,7 @@ func streamResponseXunfei2OpenAI(xunfeiResponse *ChatResponse) *openai.ChatCompl Created: helper.GetTimestamp(), Model: "SparkDesk", Choices: []openai.ChatCompletionsStreamResponseChoice{choice}, + Usage: usage, } return &response } @@ -162,6 +163,7 @@ func StreamHandler(c *gin.Context, meta *meta.Meta, textRequest model.GeneralOpe if err != nil { return openai.ErrorWrapper(err, "xunfei_request_failed", http.StatusInternalServerError), nil } + common.SetEventStreamHeaders(c) var usage model.Usage c.Stream(func(w io.Writer) bool { @@ -170,7 +172,23 @@ func StreamHandler(c *gin.Context, meta *meta.Meta, textRequest model.GeneralOpe usage.PromptTokens += xunfeiResponse.Payload.Usage.Text.PromptTokens usage.CompletionTokens += xunfeiResponse.Payload.Usage.Text.CompletionTokens usage.TotalTokens += xunfeiResponse.Payload.Usage.Text.TotalTokens - response := streamResponseXunfei2OpenAI(&xunfeiResponse) + if xunfeiResponse.Header.Code != 0 { + errMessage := fmt.Sprintf("Xunfei request failed with Sid: %s code: %d, msg: %s", xunfeiResponse.Header.Sid, xunfeiResponse.Header.Code, xunfeiResponse.Header.Message) + logger.SysError(errMessage) + mStr, err := json.Marshal(map[string]interface{}{ + "error": map[string]interface{}{ + "message": errMessage, + "code": xunfeiResponse.Header.Code, + }, + }) + if err != nil { + logger.SysError("error marshalling stream response: " + err.Error()) + return true + } + c.Render(-1, common.CustomEvent{Data: "data: " + string(mStr), Event: "error"}) + return false // 停止流式响应 + } + response := streamResponseXunfei2OpenAI(&xunfeiResponse, &usage) jsonResponse, err := json.Marshal(response) if err != nil { logger.SysError("error marshalling stream response: " + err.Error())