diff --git a/relay/channel/anthropic/adaptor.go b/relay/channel/anthropic/adaptor.go index a97ac36f..dccdd206 100644 --- a/relay/channel/anthropic/adaptor.go +++ b/relay/channel/anthropic/adaptor.go @@ -41,6 +41,8 @@ func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.G if request == nil { return nil, errors.New("request is nil") } + + c.Set("claude_model", request.Model) return ConvertRequest(*request), nil } diff --git a/relay/channel/anthropic/main.go b/relay/channel/anthropic/main.go index dced927e..4ca850db 100644 --- a/relay/channel/anthropic/main.go +++ b/relay/channel/anthropic/main.go @@ -53,14 +53,14 @@ func ConvertRequest(textRequest model.GeneralOpenAIRequest) *Request { func streamResponseClaude2OpenAI(claudeResponse *Response) *openai.ChatCompletionsStreamResponse { var choice openai.ChatCompletionsStreamResponseChoice - choice.Delta.Content = claudeResponse.Completion - finishReason := stopReasonClaude2OpenAI(claudeResponse.StopReason) + choice.Delta.Content = claudeResponse.Delta.Text + finishReason := stopReasonClaude2OpenAI(claudeResponse.Delta.StopReason) if finishReason != "null" { choice.FinishReason = &finishReason } var response openai.ChatCompletionsStreamResponse response.Object = "chat.completion.chunk" - response.Model = claudeResponse.Model + // response.Model = claudeResponse.Model response.Choices = []openai.ChatCompletionsStreamResponseChoice{choice} return &response } @@ -70,10 +70,10 @@ func responseClaude2OpenAI(claudeResponse *Response) *openai.TextResponse { Index: 0, Message: model.Message{ Role: "assistant", - Content: strings.TrimPrefix(claudeResponse.Completion, " "), + Content: strings.TrimPrefix(claudeResponse.Delta.Text, " "), Name: nil, }, - FinishReason: stopReasonClaude2OpenAI(claudeResponse.StopReason), + FinishReason: stopReasonClaude2OpenAI(claudeResponse.Delta.StopReason), } fullTextResponse := openai.TextResponse{ Id: fmt.Sprintf("chatcmpl-%s", helper.GetUUID()), @@ -121,12 +121,31 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC // some implementations may add \r at the end of data data = strings.TrimSuffix(data, "\r") var claudeResponse Response + err := json.Unmarshal([]byte(data), &claudeResponse) if err != nil { logger.SysError("error unmarshalling stream response: " + err.Error()) return true } - responseText += claudeResponse.Completion + + switch claudeResponse.Type { + case TypeContentStart, TypePing, TypeMessageDelta: + return true + case TypeContentStop, TypeMessageStop: + if claudeResponse.Delta.StopReason == "" { + claudeResponse.Delta.StopReason = "end_turn" + } + case TypeContent: + claudeResponse.Delta.StopReason = "null" + case TypeError: + logger.SysError("error response: " + claudeResponse.Error.Message) + return false + default: + logger.SysError("unknown response type: " + string(data)) + return true + } + + responseText += claudeResponse.Delta.Text response := streamResponseClaude2OpenAI(&claudeResponse) response.Id = responseId response.Created = createdTime @@ -176,7 +195,7 @@ func Handler(c *gin.Context, resp *http.Response, promptTokens int, modelName st } fullTextResponse := responseClaude2OpenAI(&claudeResponse) fullTextResponse.Model = modelName - completionTokens := openai.CountTokenText(claudeResponse.Completion, modelName) + completionTokens := openai.CountTokenText(claudeResponse.Delta.Text, modelName) usage := model.Usage{ PromptTokens: promptTokens, CompletionTokens: completionTokens, diff --git a/relay/channel/anthropic/model.go b/relay/channel/anthropic/model.go index bc718449..3efd2fef 100644 --- a/relay/channel/anthropic/model.go +++ b/relay/channel/anthropic/model.go @@ -19,9 +19,30 @@ type Error struct { Message string `json:"message"` } +type ResponseType string + +const ( + TypeError ResponseType = "error" + TypeStart ResponseType = "message_start" + TypeContentStart ResponseType = "content_block_start" + TypeContent ResponseType = "content_block_delta" + TypePing ResponseType = "ping" + TypeContentStop ResponseType = "content_block_stop" + TypeMessageDelta ResponseType = "message_delta" + TypeMessageStop ResponseType = "message_stop" +) + +// https://docs.anthropic.com/claude/reference/messages-streaming type Response struct { - Completion string `json:"completion"` - StopReason string `json:"stop_reason"` - Model string `json:"model"` - Error Error `json:"error"` + Type ResponseType `json:"type"` + Index int `json:"index,omitempty"` + Delta struct { + Type string `json:"type,omitempty"` + Text string `json:"text,omitempty"` + StopReason string `json:"stop_reason,omitempty"` + } `json:"delta,omitempty"` + Error struct { + Type string `json:"type"` + Message string `json:"message"` + } `json:"error,omitempty"` }