Merge remote-tracking branch 'remotes/origin_songquanpeng/main'

# Conflicts:
#	relay/adaptor/openai/adaptor.go
#	relay/controller/text.go

relay/adaptor/aiproxy/main.go

@@ -4,6 +4,12 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
+	"github.com/songquanpeng/one-api/common/render"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common"
 	"github.com/songquanpeng/one-api/common/helper"
@@ -12,10 +18,6 @@ import (
 	"github.com/songquanpeng/one-api/relay/adaptor/openai"
 	"github.com/songquanpeng/one-api/relay/constant"
 	"github.com/songquanpeng/one-api/relay/model"
-	"io"
-	"net/http"
-	"strconv"
-	"strings"
 )
 
 // https://docs.aiproxy.io/dev/library#使用已经定制好的知识库进行对话问答
@@ -89,6 +91,7 @@ func streamResponseAIProxyLibrary2OpenAI(response *LibraryStreamResponse) *opena
 
 func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusCode, *model.Usage) {
 	var usage model.Usage
+	var documents []LibraryDocument
 	scanner := bufio.NewScanner(resp.Body)
 	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
 		if atEOF && len(data) == 0 {
@@ -102,60 +105,48 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 		}
 		return 0, nil, nil
 	})
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
-	go func() {
-		for scanner.Scan() {
-			data := scanner.Text()
-			if len(data) < 5 { // ignore blank line or wrong format
-				continue
-			}
-			if data[:5] != "data:" {
-				continue
-			}
-			data = data[5:]
-			dataChan <- data
-		}
-		stopChan <- true
-	}()
+
 	common.SetEventStreamHeaders(c)
-	var documents []LibraryDocument
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			var AIProxyLibraryResponse LibraryStreamResponse
-			err := json.Unmarshal([]byte(data), &AIProxyLibraryResponse)
-			if err != nil {
-				logger.SysError("error unmarshalling stream response: " + err.Error())
-				return true
-			}
-			if len(AIProxyLibraryResponse.Documents) != 0 {
-				documents = AIProxyLibraryResponse.Documents
-			}
-			response := streamResponseAIProxyLibrary2OpenAI(&AIProxyLibraryResponse)
-			jsonResponse, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonResponse)})
-			return true
-		case <-stopChan:
-			response := documentsAIProxyLibrary(documents)
-			jsonResponse, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonResponse)})
-			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
-			return false
+
+	for scanner.Scan() {
+		data := scanner.Text()
+		if len(data) < 5 || data[:5] != "data:" {
+			continue
 		}
-	})
-	err := resp.Body.Close()
+		data = data[5:]
+
+		var AIProxyLibraryResponse LibraryStreamResponse
+		err := json.Unmarshal([]byte(data), &AIProxyLibraryResponse)
+		if err != nil {
+			logger.SysError("error unmarshalling stream response: " + err.Error())
+			continue
+		}
+		if len(AIProxyLibraryResponse.Documents) != 0 {
+			documents = AIProxyLibraryResponse.Documents
+		}
+		response := streamResponseAIProxyLibrary2OpenAI(&AIProxyLibraryResponse)
+		err = render.ObjectData(c, response)
+		if err != nil {
+			logger.SysError(err.Error())
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	response := documentsAIProxyLibrary(documents)
+	err := render.ObjectData(c, response)
+	if err != nil {
+		logger.SysError(err.Error())
+	}
+	render.Done(c)
+
+	err = resp.Body.Close()
 	if err != nil {
 		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil
 	}
+
 	return nil, &usage
 }
 

relay/adaptor/ali/main.go

@@ -3,15 +3,17 @@ package ali
 import (
 	"bufio"
 	"encoding/json"
+	"github.com/songquanpeng/one-api/common/render"
+	"io"
+	"net/http"
+	"strings"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common"
 	"github.com/songquanpeng/one-api/common/helper"
 	"github.com/songquanpeng/one-api/common/logger"
 	"github.com/songquanpeng/one-api/relay/adaptor/openai"
 	"github.com/songquanpeng/one-api/relay/model"
-	"io"
-	"net/http"
-	"strings"
 )
 
 // https://help.aliyun.com/document_detail/613695.html?spm=a2c4g.2399480.0.0.1adb778fAdzP9w#341800c0f8w0r
@@ -181,56 +183,43 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 		}
 		return 0, nil, nil
 	})
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
-	go func() {
-		for scanner.Scan() {
-			data := scanner.Text()
-			if len(data) < 5 { // ignore blank line or wrong format
-				continue
-			}
-			if data[:5] != "data:" {
-				continue
-			}
-			data = data[5:]
-			dataChan <- data
-		}
-		stopChan <- true
-	}()
+
 	common.SetEventStreamHeaders(c)
-	//lastResponseText := ""
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			var aliResponse ChatResponse
-			err := json.Unmarshal([]byte(data), &aliResponse)
-			if err != nil {
-				logger.SysError("error unmarshalling stream response: " + err.Error())
-				return true
-			}
-			if aliResponse.Usage.OutputTokens != 0 {
-				usage.PromptTokens = aliResponse.Usage.InputTokens
-				usage.CompletionTokens = aliResponse.Usage.OutputTokens
-				usage.TotalTokens = aliResponse.Usage.InputTokens + aliResponse.Usage.OutputTokens
-			}
-			response := streamResponseAli2OpenAI(&aliResponse)
-			if response == nil {
-				return true
-			}
-			//response.Choices[0].Delta.Content = strings.TrimPrefix(response.Choices[0].Delta.Content, lastResponseText)
-			//lastResponseText = aliResponse.Output.Text
-			jsonResponse, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonResponse)})
-			return true
-		case <-stopChan:
-			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
-			return false
+
+	for scanner.Scan() {
+		data := scanner.Text()
+		if len(data) < 5 || data[:5] != "data:" {
+			continue
 		}
-	})
+		data = data[5:]
+
+		var aliResponse ChatResponse
+		err := json.Unmarshal([]byte(data), &aliResponse)
+		if err != nil {
+			logger.SysError("error unmarshalling stream response: " + err.Error())
+			continue
+		}
+		if aliResponse.Usage.OutputTokens != 0 {
+			usage.PromptTokens = aliResponse.Usage.InputTokens
+			usage.CompletionTokens = aliResponse.Usage.OutputTokens
+			usage.TotalTokens = aliResponse.Usage.InputTokens + aliResponse.Usage.OutputTokens
+		}
+		response := streamResponseAli2OpenAI(&aliResponse)
+		if response == nil {
+			continue
+		}
+		err = render.ObjectData(c, response)
+		if err != nil {
+			logger.SysError(err.Error())
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	render.Done(c)
+
 	err := resp.Body.Close()
 	if err != nil {
 		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil

relay/adaptor/anthropic/adaptor.go

@@ -3,12 +3,14 @@ package anthropic
 import (
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/relay/adaptor"
 	"github.com/songquanpeng/one-api/relay/meta"
 	"github.com/songquanpeng/one-api/relay/model"
-	"io"
-	"net/http"
 )
 
 type Adaptor struct {
@@ -31,6 +33,13 @@ func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *me
 	}
 	req.Header.Set("anthropic-version", anthropicVersion)
 	req.Header.Set("anthropic-beta", "messages-2023-12-15")
+
+	// https://x.com/alexalbert__/status/1812921642143900036
+	// claude-3-5-sonnet can support 8k context
+	if strings.HasPrefix(meta.ActualModelName, "claude-3-5-sonnet") {
+		req.Header.Set("anthropic-beta", "max-tokens-3-5-sonnet-2024-07-15")
+	}
+
 	return nil
 }
 

relay/adaptor/anthropic/constants.go

@@ -5,4 +5,5 @@ var ModelList = []string{
 	"claude-3-haiku-20240307",
 	"claude-3-sonnet-20240229",
 	"claude-3-opus-20240229",
+	"claude-3-5-sonnet-20240620",
 }

relay/adaptor/anthropic/main.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
+	"github.com/songquanpeng/one-api/common/render"
 	"io"
 	"net/http"
 	"strings"
@@ -28,12 +29,30 @@ func stopReasonClaude2OpenAI(reason *string) string {
 		return "stop"
 	case "max_tokens":
 		return "length"
+	case "tool_use":
+		return "tool_calls"
 	default:
 		return *reason
 	}
 }
 
 func ConvertRequest(textRequest model.GeneralOpenAIRequest) *Request {
+	claudeTools := make([]Tool, 0, len(textRequest.Tools))
+
+	for _, tool := range textRequest.Tools {
+		if params, ok := tool.Function.Parameters.(map[string]any); ok {
+			claudeTools = append(claudeTools, Tool{
+				Name:        tool.Function.Name,
+				Description: tool.Function.Description,
+				InputSchema: InputSchema{
+					Type:       params["type"].(string),
+					Properties: params["properties"],
+					Required:   params["required"],
+				},
+			})
+		}
+	}
+
 	claudeRequest := Request{
 		Model:       textRequest.Model,
 		MaxTokens:   textRequest.MaxTokens,
@@ -41,6 +60,24 @@ func ConvertRequest(textRequest model.GeneralOpenAIRequest) *Request {
 		TopP:        textRequest.TopP,
 		TopK:        textRequest.TopK,
 		Stream:      textRequest.Stream,
+		Tools:       claudeTools,
+	}
+	if len(claudeTools) > 0 {
+		claudeToolChoice := struct {
+			Type string `json:"type"`
+			Name string `json:"name,omitempty"`
+		}{Type: "auto"} // default value https://docs.anthropic.com/en/docs/build-with-claude/tool-use#controlling-claudes-output
+		if choice, ok := textRequest.ToolChoice.(map[string]any); ok {
+			if function, ok := choice["function"].(map[string]any); ok {
+				claudeToolChoice.Type = "tool"
+				claudeToolChoice.Name = function["name"].(string)
+			}
+		} else if toolChoiceType, ok := textRequest.ToolChoice.(string); ok {
+			if toolChoiceType == "any" {
+				claudeToolChoice.Type = toolChoiceType
+			}
+		}
+		claudeRequest.ToolChoice = claudeToolChoice
 	}
 	if claudeRequest.MaxTokens == 0 {
 		claudeRequest.MaxTokens = 4096
@@ -63,7 +100,24 @@ func ConvertRequest(textRequest model.GeneralOpenAIRequest) *Request {
 		if message.IsStringContent() {
 			content.Type = "text"
 			content.Text = message.StringContent()
+			if message.Role == "tool" {
+				claudeMessage.Role = "user"
+				content.Type = "tool_result"
+				content.Content = content.Text
+				content.Text = ""
+				content.ToolUseId = message.ToolCallId
+			}
 			claudeMessage.Content = append(claudeMessage.Content, content)
+			for i := range message.ToolCalls {
+				inputParam := make(map[string]any)
+				_ = json.Unmarshal([]byte(message.ToolCalls[i].Function.Arguments.(string)), &inputParam)
+				claudeMessage.Content = append(claudeMessage.Content, Content{
+					Type:  "tool_use",
+					Id:    message.ToolCalls[i].Id,
+					Name:  message.ToolCalls[i].Function.Name,
+					Input: inputParam,
+				})
+			}
 			claudeRequest.Messages = append(claudeRequest.Messages, claudeMessage)
 			continue
 		}
@@ -96,16 +150,35 @@ func StreamResponseClaude2OpenAI(claudeResponse *StreamResponse) (*openai.ChatCo
 	var response *Response
 	var responseText string
 	var stopReason string
+	tools := make([]model.Tool, 0)
+
 	switch claudeResponse.Type {
 	case "message_start":
 		return nil, claudeResponse.Message
 	case "content_block_start":
 		if claudeResponse.ContentBlock != nil {
 			responseText = claudeResponse.ContentBlock.Text
+			if claudeResponse.ContentBlock.Type == "tool_use" {
+				tools = append(tools, model.Tool{
+					Id:   claudeResponse.ContentBlock.Id,
+					Type: "function",
+					Function: model.Function{
+						Name:      claudeResponse.ContentBlock.Name,
+						Arguments: "",
+					},
+				})
+			}
 		}
 	case "content_block_delta":
 		if claudeResponse.Delta != nil {
 			responseText = claudeResponse.Delta.Text
+			if claudeResponse.Delta.Type == "input_json_delta" {
+				tools = append(tools, model.Tool{
+					Function: model.Function{
+						Arguments: claudeResponse.Delta.PartialJson,
+					},
+				})
+			}
 		}
 	case "message_delta":
 		if claudeResponse.Usage != nil {
@@ -119,6 +192,10 @@ func StreamResponseClaude2OpenAI(claudeResponse *StreamResponse) (*openai.ChatCo
 	}
 	var choice openai.ChatCompletionsStreamResponseChoice
 	choice.Delta.Content = responseText
+	if len(tools) > 0 {
+		choice.Delta.Content = nil // compatible with other OpenAI derivative applications, like LobeOpenAICompatibleFactory ...
+		choice.Delta.ToolCalls = tools
+	}
 	choice.Delta.Role = "assistant"
 	finishReason := stopReasonClaude2OpenAI(&stopReason)
 	if finishReason != "null" {
@@ -135,12 +212,27 @@ func ResponseClaude2OpenAI(claudeResponse *Response) *openai.TextResponse {
 	if len(claudeResponse.Content) > 0 {
 		responseText = claudeResponse.Content[0].Text
 	}
+	tools := make([]model.Tool, 0)
+	for _, v := range claudeResponse.Content {
+		if v.Type == "tool_use" {
+			args, _ := json.Marshal(v.Input)
+			tools = append(tools, model.Tool{
+				Id:   v.Id,
+				Type: "function", // compatible with other OpenAI derivative applications
+				Function: model.Function{
+					Name:      v.Name,
+					Arguments: string(args),
+				},
+			})
+		}
+	}
 	choice := openai.TextResponseChoice{
 		Index: 0,
 		Message: model.Message{
-			Role:    "assistant",
-			Content: responseText,
-			Name:    nil,
+			Role:      "assistant",
+			Content:   responseText,
+			Name:      nil,
+			ToolCalls: tools,
 		},
 		FinishReason: stopReasonClaude2OpenAI(claudeResponse.StopReason),
 	}
@@ -169,64 +261,77 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 		}
 		return 0, nil, nil
 	})
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
-	go func() {
-		for scanner.Scan() {
-			data := scanner.Text()
-			if len(data) < 6 {
-				continue
-			}
-			if !strings.HasPrefix(data, "data:") {
-				continue
-			}
-			data = strings.TrimPrefix(data, "data:")
-			dataChan <- data
-		}
-		stopChan <- true
-	}()
+
 	common.SetEventStreamHeaders(c)
+
 	var usage model.Usage
 	var modelName string
 	var id string
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			// some implementations may add \r at the end of data
-			data = strings.TrimSpace(data)
-			var claudeResponse StreamResponse
-			err := json.Unmarshal([]byte(data), &claudeResponse)
-			if err != nil {
-				logger.SysError("error unmarshalling stream response: " + err.Error())
-				return true
-			}
-			response, meta := StreamResponseClaude2OpenAI(&claudeResponse)
-			if meta != nil {
-				usage.PromptTokens += meta.Usage.InputTokens
-				usage.CompletionTokens += meta.Usage.OutputTokens
+	var lastToolCallChoice openai.ChatCompletionsStreamResponseChoice
+
+	for scanner.Scan() {
+		data := scanner.Text()
+		if len(data) < 6 || !strings.HasPrefix(data, "data:") {
+			continue
+		}
+		data = strings.TrimPrefix(data, "data:")
+		data = strings.TrimSpace(data)
+
+		var claudeResponse StreamResponse
+		err := json.Unmarshal([]byte(data), &claudeResponse)
+		if err != nil {
+			logger.SysError("error unmarshalling stream response: " + err.Error())
+			continue
+		}
+
+		response, meta := StreamResponseClaude2OpenAI(&claudeResponse)
+		if meta != nil {
+			usage.PromptTokens += meta.Usage.InputTokens
+			usage.CompletionTokens += meta.Usage.OutputTokens
+			if len(meta.Id) > 0 { // only message_start has an id, otherwise it's a finish_reason event.
 				modelName = meta.Model
 				id = fmt.Sprintf("chatcmpl-%s", meta.Id)
-				return true
+				continue
+			} else { // finish_reason case
+				if len(lastToolCallChoice.Delta.ToolCalls) > 0 {
+					lastArgs := &lastToolCallChoice.Delta.ToolCalls[len(lastToolCallChoice.Delta.ToolCalls)-1].Function
+					if len(lastArgs.Arguments.(string)) == 0 { // compatible with OpenAI sending an empty object `{}` when no arguments.
+						lastArgs.Arguments = "{}"
+						response.Choices[len(response.Choices)-1].Delta.Content = nil
+						response.Choices[len(response.Choices)-1].Delta.ToolCalls = lastToolCallChoice.Delta.ToolCalls
+					}
+				}
 			}
-			if response == nil {
-				return true
-			}
-			response.Id = id
-			response.Model = modelName
-			response.Created = createdTime
-			jsonStr, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonStr)})
-			return true
-		case <-stopChan:
-			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
-			return false
 		}
-	})
-	_ = resp.Body.Close()
+		if response == nil {
+			continue
+		}
+
+		response.Id = id
+		response.Model = modelName
+		response.Created = createdTime
+
+		for _, choice := range response.Choices {
+			if len(choice.Delta.ToolCalls) > 0 {
+				lastToolCallChoice = choice
+			}
+		}
+		err = render.ObjectData(c, response)
+		if err != nil {
+			logger.SysError(err.Error())
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	render.Done(c)
+
+	err := resp.Body.Close()
+	if err != nil {
+		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil
+	}
 	return nil, &usage
 }
 

relay/adaptor/anthropic/model.go

@@ -16,6 +16,12 @@ type Content struct {
 	Type   string       `json:"type"`
 	Text   string       `json:"text,omitempty"`
 	Source *ImageSource `json:"source,omitempty"`
+	// tool_calls
+	Id        string `json:"id,omitempty"`
+	Name      string `json:"name,omitempty"`
+	Input     any    `json:"input,omitempty"`
+	Content   string `json:"content,omitempty"`
+	ToolUseId string `json:"tool_use_id,omitempty"`
 }
 
 type Message struct {
@@ -23,6 +29,18 @@ type Message struct {
 	Content []Content `json:"content"`
 }
 
+type Tool struct {
+	Name        string      `json:"name"`
+	Description string      `json:"description,omitempty"`
+	InputSchema InputSchema `json:"input_schema"`
+}
+
+type InputSchema struct {
+	Type       string `json:"type"`
+	Properties any    `json:"properties,omitempty"`
+	Required   any    `json:"required,omitempty"`
+}
+
 type Request struct {
 	Model         string    `json:"model"`
 	Messages      []Message `json:"messages"`
@@ -33,6 +51,8 @@ type Request struct {
 	Temperature   float64   `json:"temperature,omitempty"`
 	TopP          float64   `json:"top_p,omitempty"`
 	TopK          int       `json:"top_k,omitempty"`
+	Tools         []Tool    `json:"tools,omitempty"`
+	ToolChoice    any       `json:"tool_choice,omitempty"`
 	//Metadata    `json:"metadata,omitempty"`
 }
 
@@ -61,6 +81,7 @@ type Response struct {
 type Delta struct {
 	Type         string  `json:"type"`
 	Text         string  `json:"text"`
+	PartialJson  string  `json:"partial_json,omitempty"`
 	StopReason   *string `json:"stop_reason"`
 	StopSequence *string `json:"stop_sequence"`
 }

relay/adaptor/aws/adaptor.go

@@ -1,17 +1,16 @@
 package aws
 
 import (
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
-	"github.com/songquanpeng/one-api/common/ctxkey"
+	"errors"
 	"io"
 	"net/http"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
 	"github.com/gin-gonic/gin"
-	"github.com/pkg/errors"
 	"github.com/songquanpeng/one-api/relay/adaptor"
-	"github.com/songquanpeng/one-api/relay/adaptor/anthropic"
+	"github.com/songquanpeng/one-api/relay/adaptor/aws/utils"
 	"github.com/songquanpeng/one-api/relay/meta"
 	"github.com/songquanpeng/one-api/relay/model"
 )
@@ -19,18 +18,52 @@ import (
 var _ adaptor.Adaptor = new(Adaptor)
 
 type Adaptor struct {
-	meta      *meta.Meta
-	awsClient *bedrockruntime.Client
+	awsAdapter utils.AwsAdapter
+
+	Meta      *meta.Meta
+	AwsClient *bedrockruntime.Client
 }
 
 func (a *Adaptor) Init(meta *meta.Meta) {
-	a.meta = meta
-	a.awsClient = bedrockruntime.New(bedrockruntime.Options{
+	a.Meta = meta
+	a.AwsClient = bedrockruntime.New(bedrockruntime.Options{
 		Region:      meta.Config.Region,
 		Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(meta.Config.AK, meta.Config.SK, "")),
 	})
 }
 
+func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error) {
+	if request == nil {
+		return nil, errors.New("request is nil")
+	}
+
+	adaptor := GetAdaptor(request.Model)
+	if adaptor == nil {
+		return nil, errors.New("adaptor not found")
+	}
+
+	a.awsAdapter = adaptor
+	return adaptor.ConvertRequest(c, relayMode, request)
+}
+
+func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
+	if a.awsAdapter == nil {
+		return nil, utils.WrapErr(errors.New("awsAdapter is nil"))
+	}
+	return a.awsAdapter.DoResponse(c, a.AwsClient, meta)
+}
+
+func (a *Adaptor) GetModelList() (models []string) {
+	for model := range adaptors {
+		models = append(models, model)
+	}
+	return
+}
+
+func (a *Adaptor) GetChannelName() string {
+	return "aws"
+}
+
 func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
 	return "", nil
 }
@@ -39,17 +72,6 @@ func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *me
 	return nil
 }
 
-func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error) {
-	if request == nil {
-		return nil, errors.New("request is nil")
-	}
-
-	claudeReq := anthropic.ConvertRequest(*request)
-	c.Set(ctxkey.RequestModel, request.Model)
-	c.Set(ctxkey.ConvertedRequest, claudeReq)
-	return claudeReq, nil
-}
-
 func (a *Adaptor) ConvertImageRequest(request *model.ImageRequest) (any, error) {
 	if request == nil {
 		return nil, errors.New("request is nil")
@@ -60,23 +82,3 @@ func (a *Adaptor) ConvertImageRequest(request *model.ImageRequest) (any, error)
 func (a *Adaptor) DoRequest(c *gin.Context, meta *meta.Meta, requestBody io.Reader) (*http.Response, error) {
 	return nil, nil
 }
-
-func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
-	if meta.IsStream {
-		err, usage = StreamHandler(c, a.awsClient)
-	} else {
-		err, usage = Handler(c, a.awsClient, meta.ActualModelName)
-	}
-	return
-}
-
-func (a *Adaptor) GetModelList() (models []string) {
-	for n := range awsModelIDMap {
-		models = append(models, n)
-	}
-	return
-}
-
-func (a *Adaptor) GetChannelName() string {
-	return "aws"
-}

relay/adaptor/aws/claude/adapter.go

@@ -0,0 +1,37 @@
+package aws
+
+import (
+	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
+	"github.com/gin-gonic/gin"
+	"github.com/pkg/errors"
+	"github.com/songquanpeng/one-api/common/ctxkey"
+	"github.com/songquanpeng/one-api/relay/adaptor/anthropic"
+	"github.com/songquanpeng/one-api/relay/adaptor/aws/utils"
+	"github.com/songquanpeng/one-api/relay/meta"
+	"github.com/songquanpeng/one-api/relay/model"
+)
+
+var _ utils.AwsAdapter = new(Adaptor)
+
+type Adaptor struct {
+}
+
+func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error) {
+	if request == nil {
+		return nil, errors.New("request is nil")
+	}
+
+	claudeReq := anthropic.ConvertRequest(*request)
+	c.Set(ctxkey.RequestModel, request.Model)
+	c.Set(ctxkey.ConvertedRequest, claudeReq)
+	return claudeReq, nil
+}
+
+func (a *Adaptor) DoResponse(c *gin.Context, awsCli *bedrockruntime.Client, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
+	if meta.IsStream {
+		err, usage = StreamHandler(c, awsCli)
+	} else {
+		err, usage = Handler(c, awsCli, meta.ActualModelName)
+	}
+	return
+}

relay/adaptor/aws/claude/main.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/songquanpeng/one-api/common/ctxkey"
 	"io"
 	"net/http"
 
@@ -16,33 +15,28 @@ import (
 	"github.com/jinzhu/copier"
 	"github.com/pkg/errors"
 	"github.com/songquanpeng/one-api/common"
+	"github.com/songquanpeng/one-api/common/ctxkey"
 	"github.com/songquanpeng/one-api/common/helper"
 	"github.com/songquanpeng/one-api/common/logger"
 	"github.com/songquanpeng/one-api/relay/adaptor/anthropic"
+	"github.com/songquanpeng/one-api/relay/adaptor/aws/utils"
+	"github.com/songquanpeng/one-api/relay/adaptor/openai"
 	relaymodel "github.com/songquanpeng/one-api/relay/model"
 )
 
-func wrapErr(err error) *relaymodel.ErrorWithStatusCode {
-	return &relaymodel.ErrorWithStatusCode{
-		StatusCode: http.StatusInternalServerError,
-		Error: relaymodel.Error{
-			Message: fmt.Sprintf("%s", err.Error()),
-		},
-	}
-}
-
 // https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html
-var awsModelIDMap = map[string]string{
-	"claude-instant-1.2":       "anthropic.claude-instant-v1",
-	"claude-2.0":               "anthropic.claude-v2",
-	"claude-2.1":               "anthropic.claude-v2:1",
-	"claude-3-sonnet-20240229": "anthropic.claude-3-sonnet-20240229-v1:0",
-	"claude-3-opus-20240229":   "anthropic.claude-3-opus-20240229-v1:0",
-	"claude-3-haiku-20240307":  "anthropic.claude-3-haiku-20240307-v1:0",
+var AwsModelIDMap = map[string]string{
+	"claude-instant-1.2":         "anthropic.claude-instant-v1",
+	"claude-2.0":                 "anthropic.claude-v2",
+	"claude-2.1":                 "anthropic.claude-v2:1",
+	"claude-3-sonnet-20240229":   "anthropic.claude-3-sonnet-20240229-v1:0",
+	"claude-3-5-sonnet-20240620": "anthropic.claude-3-5-sonnet-20240620-v1:0",
+	"claude-3-opus-20240229":     "anthropic.claude-3-opus-20240229-v1:0",
+	"claude-3-haiku-20240307":    "anthropic.claude-3-haiku-20240307-v1:0",
 }
 
 func awsModelID(requestModel string) (string, error) {
-	if awsModelID, ok := awsModelIDMap[requestModel]; ok {
+	if awsModelID, ok := AwsModelIDMap[requestModel]; ok {
 		return awsModelID, nil
 	}
 
@@ -52,7 +46,7 @@ func awsModelID(requestModel string) (string, error) {
 func Handler(c *gin.Context, awsCli *bedrockruntime.Client, modelName string) (*relaymodel.ErrorWithStatusCode, *relaymodel.Usage) {
 	awsModelId, err := awsModelID(c.GetString(ctxkey.RequestModel))
 	if err != nil {
-		return wrapErr(errors.Wrap(err, "awsModelID")), nil
+		return utils.WrapErr(errors.Wrap(err, "awsModelID")), nil
 	}
 
 	awsReq := &bedrockruntime.InvokeModelInput{
@@ -63,30 +57,30 @@ func Handler(c *gin.Context, awsCli *bedrockruntime.Client, modelName string) (*
 
 	claudeReq_, ok := c.Get(ctxkey.ConvertedRequest)
 	if !ok {
-		return wrapErr(errors.New("request not found")), nil
+		return utils.WrapErr(errors.New("request not found")), nil
 	}
 	claudeReq := claudeReq_.(*anthropic.Request)
 	awsClaudeReq := &Request{
 		AnthropicVersion: "bedrock-2023-05-31",
 	}
 	if err = copier.Copy(awsClaudeReq, claudeReq); err != nil {
-		return wrapErr(errors.Wrap(err, "copy request")), nil
+		return utils.WrapErr(errors.Wrap(err, "copy request")), nil
 	}
 
 	awsReq.Body, err = json.Marshal(awsClaudeReq)
 	if err != nil {
-		return wrapErr(errors.Wrap(err, "marshal request")), nil
+		return utils.WrapErr(errors.Wrap(err, "marshal request")), nil
 	}
 
 	awsResp, err := awsCli.InvokeModel(c.Request.Context(), awsReq)
 	if err != nil {
-		return wrapErr(errors.Wrap(err, "InvokeModel")), nil
+		return utils.WrapErr(errors.Wrap(err, "InvokeModel")), nil
 	}
 
 	claudeResponse := new(anthropic.Response)
 	err = json.Unmarshal(awsResp.Body, claudeResponse)
 	if err != nil {
-		return wrapErr(errors.Wrap(err, "unmarshal response")), nil
+		return utils.WrapErr(errors.Wrap(err, "unmarshal response")), nil
 	}
 
 	openaiResp := anthropic.ResponseClaude2OpenAI(claudeResponse)
@@ -106,7 +100,7 @@ func StreamHandler(c *gin.Context, awsCli *bedrockruntime.Client) (*relaymodel.E
 	createdTime := helper.GetTimestamp()
 	awsModelId, err := awsModelID(c.GetString(ctxkey.RequestModel))
 	if err != nil {
-		return wrapErr(errors.Wrap(err, "awsModelID")), nil
+		return utils.WrapErr(errors.Wrap(err, "awsModelID")), nil
 	}
 
 	awsReq := &bedrockruntime.InvokeModelWithResponseStreamInput{
@@ -117,7 +111,7 @@ func StreamHandler(c *gin.Context, awsCli *bedrockruntime.Client) (*relaymodel.E
 
 	claudeReq_, ok := c.Get(ctxkey.ConvertedRequest)
 	if !ok {
-		return wrapErr(errors.New("request not found")), nil
+		return utils.WrapErr(errors.New("request not found")), nil
 	}
 	claudeReq := claudeReq_.(*anthropic.Request)
 
@@ -125,16 +119,16 @@ func StreamHandler(c *gin.Context, awsCli *bedrockruntime.Client) (*relaymodel.E
 		AnthropicVersion: "bedrock-2023-05-31",
 	}
 	if err = copier.Copy(awsClaudeReq, claudeReq); err != nil {
-		return wrapErr(errors.Wrap(err, "copy request")), nil
+		return utils.WrapErr(errors.Wrap(err, "copy request")), nil
 	}
 	awsReq.Body, err = json.Marshal(awsClaudeReq)
 	if err != nil {
-		return wrapErr(errors.Wrap(err, "marshal request")), nil
+		return utils.WrapErr(errors.Wrap(err, "marshal request")), nil
 	}
 
 	awsResp, err := awsCli.InvokeModelWithResponseStream(c.Request.Context(), awsReq)
 	if err != nil {
-		return wrapErr(errors.Wrap(err, "InvokeModelWithResponseStream")), nil
+		return utils.WrapErr(errors.Wrap(err, "InvokeModelWithResponseStream")), nil
 	}
 	stream := awsResp.GetStream()
 	defer stream.Close()
@@ -142,6 +136,8 @@ func StreamHandler(c *gin.Context, awsCli *bedrockruntime.Client) (*relaymodel.E
 	c.Writer.Header().Set("Content-Type", "text/event-stream")
 	var usage relaymodel.Usage
 	var id string
+	var lastToolCallChoice openai.ChatCompletionsStreamResponseChoice
+
 	c.Stream(func(w io.Writer) bool {
 		event, ok := <-stream.Events()
 		if !ok {
@@ -162,8 +158,19 @@ func StreamHandler(c *gin.Context, awsCli *bedrockruntime.Client) (*relaymodel.E
 			if meta != nil {
 				usage.PromptTokens += meta.Usage.InputTokens
 				usage.CompletionTokens += meta.Usage.OutputTokens
-				id = fmt.Sprintf("chatcmpl-%s", meta.Id)
-				return true
+				if len(meta.Id) > 0 { // only message_start has an id, otherwise it's a finish_reason event.
+					id = fmt.Sprintf("chatcmpl-%s", meta.Id)
+					return true
+				} else { // finish_reason case
+					if len(lastToolCallChoice.Delta.ToolCalls) > 0 {
+						lastArgs := &lastToolCallChoice.Delta.ToolCalls[len(lastToolCallChoice.Delta.ToolCalls)-1].Function
+						if len(lastArgs.Arguments.(string)) == 0 { // compatible with OpenAI sending an empty object `{}` when no arguments.
+							lastArgs.Arguments = "{}"
+							response.Choices[len(response.Choices)-1].Delta.Content = nil
+							response.Choices[len(response.Choices)-1].Delta.ToolCalls = lastToolCallChoice.Delta.ToolCalls
+						}
+					}
+				}
 			}
 			if response == nil {
 				return true
@@ -171,6 +178,12 @@ func StreamHandler(c *gin.Context, awsCli *bedrockruntime.Client) (*relaymodel.E
 			response.Id = id
 			response.Model = c.GetString(ctxkey.OriginalModel)
 			response.Created = createdTime
+
+			for _, choice := range response.Choices {
+				if len(choice.Delta.ToolCalls) > 0 {
+					lastToolCallChoice = choice
+				}
+			}
 			jsonStr, err := json.Marshal(response)
 			if err != nil {
 				logger.SysError("error marshalling stream response: " + err.Error())

relay/adaptor/aws/claude/model.go

@@ -9,9 +9,12 @@ type Request struct {
 	// AnthropicVersion should be "bedrock-2023-05-31"
 	AnthropicVersion string              `json:"anthropic_version"`
 	Messages         []anthropic.Message `json:"messages"`
+	System           string              `json:"system,omitempty"`
 	MaxTokens        int                 `json:"max_tokens,omitempty"`
 	Temperature      float64             `json:"temperature,omitempty"`
 	TopP             float64             `json:"top_p,omitempty"`
 	TopK             int                 `json:"top_k,omitempty"`
 	StopSequences    []string            `json:"stop_sequences,omitempty"`
+	Tools            []anthropic.Tool    `json:"tools,omitempty"`
+	ToolChoice       any                 `json:"tool_choice,omitempty"`
 }

relay/adaptor/aws/llama3/adapter.go

@@ -0,0 +1,37 @@
+package aws
+
+import (
+	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
+	"github.com/songquanpeng/one-api/common/ctxkey"
+
+	"github.com/gin-gonic/gin"
+	"github.com/pkg/errors"
+	"github.com/songquanpeng/one-api/relay/adaptor/aws/utils"
+	"github.com/songquanpeng/one-api/relay/meta"
+	"github.com/songquanpeng/one-api/relay/model"
+)
+
+var _ utils.AwsAdapter = new(Adaptor)
+
+type Adaptor struct {
+}
+
+func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error) {
+	if request == nil {
+		return nil, errors.New("request is nil")
+	}
+
+	llamaReq := ConvertRequest(*request)
+	c.Set(ctxkey.RequestModel, request.Model)
+	c.Set(ctxkey.ConvertedRequest, llamaReq)
+	return llamaReq, nil
+}
+
+func (a *Adaptor) DoResponse(c *gin.Context, awsCli *bedrockruntime.Client, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
+	if meta.IsStream {
+		err, usage = StreamHandler(c, awsCli)
+	} else {
+		err, usage = Handler(c, awsCli, meta.ActualModelName)
+	}
+	return
+}

relay/adaptor/aws/llama3/main.go

@@ -0,0 +1,231 @@
+// Package aws provides the AWS adaptor for the relay service.
+package aws
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"text/template"
+
+	"github.com/songquanpeng/one-api/common/ctxkey"
+	"github.com/songquanpeng/one-api/common/random"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
+	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"
+	"github.com/gin-gonic/gin"
+	"github.com/pkg/errors"
+	"github.com/songquanpeng/one-api/common"
+	"github.com/songquanpeng/one-api/common/helper"
+	"github.com/songquanpeng/one-api/common/logger"
+	"github.com/songquanpeng/one-api/relay/adaptor/aws/utils"
+	"github.com/songquanpeng/one-api/relay/adaptor/openai"
+	relaymodel "github.com/songquanpeng/one-api/relay/model"
+)
+
+// Only support llama-3-8b and llama-3-70b instruction models
+// https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html
+var AwsModelIDMap = map[string]string{
+	"llama3-8b-8192":  "meta.llama3-8b-instruct-v1:0",
+	"llama3-70b-8192": "meta.llama3-70b-instruct-v1:0",
+}
+
+func awsModelID(requestModel string) (string, error) {
+	if awsModelID, ok := AwsModelIDMap[requestModel]; ok {
+		return awsModelID, nil
+	}
+
+	return "", errors.Errorf("model %s not found", requestModel)
+}
+
+// promptTemplate with range
+const promptTemplate = `<|begin_of_text|>{{range .Messages}}<|start_header_id|>{{.Role}}<|end_header_id|>{{.StringContent}}<|eot_id|>{{end}}<|start_header_id|>assistant<|end_header_id|>
+`
+
+var promptTpl = template.Must(template.New("llama3-chat").Parse(promptTemplate))
+
+func RenderPrompt(messages []relaymodel.Message) string {
+	var buf bytes.Buffer
+	err := promptTpl.Execute(&buf, struct{ Messages []relaymodel.Message }{messages})
+	if err != nil {
+		logger.SysError("error rendering prompt messages: " + err.Error())
+	}
+	return buf.String()
+}
+
+func ConvertRequest(textRequest relaymodel.GeneralOpenAIRequest) *Request {
+	llamaRequest := Request{
+		MaxGenLen:   textRequest.MaxTokens,
+		Temperature: textRequest.Temperature,
+		TopP:        textRequest.TopP,
+	}
+	if llamaRequest.MaxGenLen == 0 {
+		llamaRequest.MaxGenLen = 2048
+	}
+	prompt := RenderPrompt(textRequest.Messages)
+	llamaRequest.Prompt = prompt
+	return &llamaRequest
+}
+
+func Handler(c *gin.Context, awsCli *bedrockruntime.Client, modelName string) (*relaymodel.ErrorWithStatusCode, *relaymodel.Usage) {
+	awsModelId, err := awsModelID(c.GetString(ctxkey.RequestModel))
+	if err != nil {
+		return utils.WrapErr(errors.Wrap(err, "awsModelID")), nil
+	}
+
+	awsReq := &bedrockruntime.InvokeModelInput{
+		ModelId:     aws.String(awsModelId),
+		Accept:      aws.String("application/json"),
+		ContentType: aws.String("application/json"),
+	}
+
+	llamaReq, ok := c.Get(ctxkey.ConvertedRequest)
+	if !ok {
+		return utils.WrapErr(errors.New("request not found")), nil
+	}
+
+	awsReq.Body, err = json.Marshal(llamaReq)
+	if err != nil {
+		return utils.WrapErr(errors.Wrap(err, "marshal request")), nil
+	}
+
+	awsResp, err := awsCli.InvokeModel(c.Request.Context(), awsReq)
+	if err != nil {
+		return utils.WrapErr(errors.Wrap(err, "InvokeModel")), nil
+	}
+
+	var llamaResponse Response
+	err = json.Unmarshal(awsResp.Body, &llamaResponse)
+	if err != nil {
+		return utils.WrapErr(errors.Wrap(err, "unmarshal response")), nil
+	}
+
+	openaiResp := ResponseLlama2OpenAI(&llamaResponse)
+	openaiResp.Model = modelName
+	usage := relaymodel.Usage{
+		PromptTokens:     llamaResponse.PromptTokenCount,
+		CompletionTokens: llamaResponse.GenerationTokenCount,
+		TotalTokens:      llamaResponse.PromptTokenCount + llamaResponse.GenerationTokenCount,
+	}
+	openaiResp.Usage = usage
+
+	c.JSON(http.StatusOK, openaiResp)
+	return nil, &usage
+}
+
+func ResponseLlama2OpenAI(llamaResponse *Response) *openai.TextResponse {
+	var responseText string
+	if len(llamaResponse.Generation) > 0 {
+		responseText = llamaResponse.Generation
+	}
+	choice := openai.TextResponseChoice{
+		Index: 0,
+		Message: relaymodel.Message{
+			Role:    "assistant",
+			Content: responseText,
+			Name:    nil,
+		},
+		FinishReason: llamaResponse.StopReason,
+	}
+	fullTextResponse := openai.TextResponse{
+		Id:      fmt.Sprintf("chatcmpl-%s", random.GetUUID()),
+		Object:  "chat.completion",
+		Created: helper.GetTimestamp(),
+		Choices: []openai.TextResponseChoice{choice},
+	}
+	return &fullTextResponse
+}
+
+func StreamHandler(c *gin.Context, awsCli *bedrockruntime.Client) (*relaymodel.ErrorWithStatusCode, *relaymodel.Usage) {
+	createdTime := helper.GetTimestamp()
+	awsModelId, err := awsModelID(c.GetString(ctxkey.RequestModel))
+	if err != nil {
+		return utils.WrapErr(errors.Wrap(err, "awsModelID")), nil
+	}
+
+	awsReq := &bedrockruntime.InvokeModelWithResponseStreamInput{
+		ModelId:     aws.String(awsModelId),
+		Accept:      aws.String("application/json"),
+		ContentType: aws.String("application/json"),
+	}
+
+	llamaReq, ok := c.Get(ctxkey.ConvertedRequest)
+	if !ok {
+		return utils.WrapErr(errors.New("request not found")), nil
+	}
+
+	awsReq.Body, err = json.Marshal(llamaReq)
+	if err != nil {
+		return utils.WrapErr(errors.Wrap(err, "marshal request")), nil
+	}
+
+	awsResp, err := awsCli.InvokeModelWithResponseStream(c.Request.Context(), awsReq)
+	if err != nil {
+		return utils.WrapErr(errors.Wrap(err, "InvokeModelWithResponseStream")), nil
+	}
+	stream := awsResp.GetStream()
+	defer stream.Close()
+
+	c.Writer.Header().Set("Content-Type", "text/event-stream")
+	var usage relaymodel.Usage
+	c.Stream(func(w io.Writer) bool {
+		event, ok := <-stream.Events()
+		if !ok {
+			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
+			return false
+		}
+
+		switch v := event.(type) {
+		case *types.ResponseStreamMemberChunk:
+			var llamaResp StreamResponse
+			err := json.NewDecoder(bytes.NewReader(v.Value.Bytes)).Decode(&llamaResp)
+			if err != nil {
+				logger.SysError("error unmarshalling stream response: " + err.Error())
+				return false
+			}
+
+			if llamaResp.PromptTokenCount > 0 {
+				usage.PromptTokens = llamaResp.PromptTokenCount
+			}
+			if llamaResp.StopReason == "stop" {
+				usage.CompletionTokens = llamaResp.GenerationTokenCount
+				usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens
+			}
+			response := StreamResponseLlama2OpenAI(&llamaResp)
+			response.Id = fmt.Sprintf("chatcmpl-%s", random.GetUUID())
+			response.Model = c.GetString(ctxkey.OriginalModel)
+			response.Created = createdTime
+			jsonStr, err := json.Marshal(response)
+			if err != nil {
+				logger.SysError("error marshalling stream response: " + err.Error())
+				return true
+			}
+			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonStr)})
+			return true
+		case *types.UnknownUnionMember:
+			fmt.Println("unknown tag:", v.Tag)
+			return false
+		default:
+			fmt.Println("union is nil or unknown type")
+			return false
+		}
+	})
+
+	return nil, &usage
+}
+
+func StreamResponseLlama2OpenAI(llamaResponse *StreamResponse) *openai.ChatCompletionsStreamResponse {
+	var choice openai.ChatCompletionsStreamResponseChoice
+	choice.Delta.Content = llamaResponse.Generation
+	choice.Delta.Role = "assistant"
+	finishReason := llamaResponse.StopReason
+	if finishReason != "null" {
+		choice.FinishReason = &finishReason
+	}
+	var openaiResponse openai.ChatCompletionsStreamResponse
+	openaiResponse.Object = "chat.completion.chunk"
+	openaiResponse.Choices = []openai.ChatCompletionsStreamResponseChoice{choice}
+	return &openaiResponse
+}

relay/adaptor/aws/llama3/main_test.go

@@ -0,0 +1,45 @@
+package aws_test
+
+import (
+	"testing"
+
+	aws "github.com/songquanpeng/one-api/relay/adaptor/aws/llama3"
+	relaymodel "github.com/songquanpeng/one-api/relay/model"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRenderPrompt(t *testing.T) {
+	messages := []relaymodel.Message{
+		{
+			Role:    "user",
+			Content: "What's your name?",
+		},
+	}
+	prompt := aws.RenderPrompt(messages)
+	expected := `<|begin_of_text|><|start_header_id|>user<|end_header_id|>What's your name?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+`
+	assert.Equal(t, expected, prompt)
+
+	messages = []relaymodel.Message{
+		{
+			Role:    "system",
+			Content: "Your name is Kat. You are a detective.",
+		},
+		{
+			Role:    "user",
+			Content: "What's your name?",
+		},
+		{
+			Role:    "assistant",
+			Content: "Kat",
+		},
+		{
+			Role:    "user",
+			Content: "What's your job?",
+		},
+	}
+	prompt = aws.RenderPrompt(messages)
+	expected = `<|begin_of_text|><|start_header_id|>system<|end_header_id|>Your name is Kat. You are a detective.<|eot_id|><|start_header_id|>user<|end_header_id|>What's your name?<|eot_id|><|start_header_id|>assistant<|end_header_id|>Kat<|eot_id|><|start_header_id|>user<|end_header_id|>What's your job?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+`
+	assert.Equal(t, expected, prompt)
+}

relay/adaptor/aws/llama3/model.go

@@ -0,0 +1,29 @@
+package aws
+
+// Request is the request to AWS Llama3
+//
+// https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-meta.html
+type Request struct {
+	Prompt      string  `json:"prompt"`
+	MaxGenLen   int     `json:"max_gen_len,omitempty"`
+	Temperature float64 `json:"temperature,omitempty"`
+	TopP        float64 `json:"top_p,omitempty"`
+}
+
+// Response is the response from AWS Llama3
+//
+// https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-meta.html
+type Response struct {
+	Generation           string `json:"generation"`
+	PromptTokenCount     int    `json:"prompt_token_count"`
+	GenerationTokenCount int    `json:"generation_token_count"`
+	StopReason           string `json:"stop_reason"`
+}
+
+// {'generation': 'Hi', 'prompt_token_count': 15, 'generation_token_count': 1, 'stop_reason': None}
+type StreamResponse struct {
+	Generation           string `json:"generation"`
+	PromptTokenCount     int    `json:"prompt_token_count"`
+	GenerationTokenCount int    `json:"generation_token_count"`
+	StopReason           string `json:"stop_reason"`
+}

relay/adaptor/aws/registry.go

@@ -0,0 +1,39 @@
+package aws
+
+import (
+	claude "github.com/songquanpeng/one-api/relay/adaptor/aws/claude"
+	llama3 "github.com/songquanpeng/one-api/relay/adaptor/aws/llama3"
+	"github.com/songquanpeng/one-api/relay/adaptor/aws/utils"
+)
+
+type AwsModelType int
+
+const (
+	AwsClaude AwsModelType = iota + 1
+	AwsLlama3
+)
+
+var (
+	adaptors = map[string]AwsModelType{}
+)
+
+func init() {
+	for model := range claude.AwsModelIDMap {
+		adaptors[model] = AwsClaude
+	}
+	for model := range llama3.AwsModelIDMap {
+		adaptors[model] = AwsLlama3
+	}
+}
+
+func GetAdaptor(model string) utils.AwsAdapter {
+	adaptorType := adaptors[model]
+	switch adaptorType {
+	case AwsClaude:
+		return &claude.Adaptor{}
+	case AwsLlama3:
+		return &llama3.Adaptor{}
+	default:
+		return nil
+	}
+}

relay/adaptor/aws/utils/adaptor.go

@@ -0,0 +1,51 @@
+package utils
+
+import (
+	"errors"
+	"io"
+	"net/http"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
+	"github.com/gin-gonic/gin"
+	"github.com/songquanpeng/one-api/relay/meta"
+	"github.com/songquanpeng/one-api/relay/model"
+)
+
+type AwsAdapter interface {
+	ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error)
+	DoResponse(c *gin.Context, awsCli *bedrockruntime.Client, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode)
+}
+
+type Adaptor struct {
+	Meta      *meta.Meta
+	AwsClient *bedrockruntime.Client
+}
+
+func (a *Adaptor) Init(meta *meta.Meta) {
+	a.Meta = meta
+	a.AwsClient = bedrockruntime.New(bedrockruntime.Options{
+		Region:      meta.Config.Region,
+		Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(meta.Config.AK, meta.Config.SK, "")),
+	})
+}
+
+func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
+	return "", nil
+}
+
+func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *meta.Meta) error {
+	return nil
+}
+
+func (a *Adaptor) ConvertImageRequest(request *model.ImageRequest) (any, error) {
+	if request == nil {
+		return nil, errors.New("request is nil")
+	}
+	return request, nil
+}
+
+func (a *Adaptor) DoRequest(c *gin.Context, meta *meta.Meta, requestBody io.Reader) (*http.Response, error) {
+	return nil, nil
+}

relay/adaptor/aws/utils/utils.go

@@ -0,0 +1,16 @@
+package utils
+
+import (
+	"net/http"
+
+	relaymodel "github.com/songquanpeng/one-api/relay/model"
+)
+
+func WrapErr(err error) *relaymodel.ErrorWithStatusCode {
+	return &relaymodel.ErrorWithStatusCode{
+		StatusCode: http.StatusInternalServerError,
+		Error: relaymodel.Error{
+			Message: err.Error(),
+		},
+	}
+}

relay/adaptor/baidu/main.go

@@ -5,6 +5,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/songquanpeng/one-api/common/render"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common"
 	"github.com/songquanpeng/one-api/common/client"
@@ -12,11 +19,6 @@ import (
 	"github.com/songquanpeng/one-api/relay/adaptor/openai"
 	"github.com/songquanpeng/one-api/relay/constant"
 	"github.com/songquanpeng/one-api/relay/model"
-	"io"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
 )
 
 // https://cloud.baidu.com/doc/WENXINWORKSHOP/s/flfmc9do2
@@ -137,59 +139,41 @@ func embeddingResponseBaidu2OpenAI(response *EmbeddingResponse) *openai.Embeddin
 func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusCode, *model.Usage) {
 	var usage model.Usage
 	scanner := bufio.NewScanner(resp.Body)
-	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
-		if atEOF && len(data) == 0 {
-			return 0, nil, nil
-		}
-		if i := strings.Index(string(data), "\n"); i >= 0 {
-			return i + 1, data[0:i], nil
-		}
-		if atEOF {
-			return len(data), data, nil
-		}
-		return 0, nil, nil
-	})
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
-	go func() {
-		for scanner.Scan() {
-			data := scanner.Text()
-			if len(data) < 6 { // ignore blank line or wrong format
-				continue
-			}
-			data = data[6:]
-			dataChan <- data
-		}
-		stopChan <- true
-	}()
+	scanner.Split(bufio.ScanLines)
+
 	common.SetEventStreamHeaders(c)
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			var baiduResponse ChatStreamResponse
-			err := json.Unmarshal([]byte(data), &baiduResponse)
-			if err != nil {
-				logger.SysError("error unmarshalling stream response: " + err.Error())
-				return true
-			}
-			if baiduResponse.Usage.TotalTokens != 0 {
-				usage.TotalTokens = baiduResponse.Usage.TotalTokens
-				usage.PromptTokens = baiduResponse.Usage.PromptTokens
-				usage.CompletionTokens = baiduResponse.Usage.TotalTokens - baiduResponse.Usage.PromptTokens
-			}
-			response := streamResponseBaidu2OpenAI(&baiduResponse)
-			jsonResponse, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonResponse)})
-			return true
-		case <-stopChan:
-			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
-			return false
+
+	for scanner.Scan() {
+		data := scanner.Text()
+		if len(data) < 6 {
+			continue
 		}
-	})
+		data = data[6:]
+
+		var baiduResponse ChatStreamResponse
+		err := json.Unmarshal([]byte(data), &baiduResponse)
+		if err != nil {
+			logger.SysError("error unmarshalling stream response: " + err.Error())
+			continue
+		}
+		if baiduResponse.Usage.TotalTokens != 0 {
+			usage.TotalTokens = baiduResponse.Usage.TotalTokens
+			usage.PromptTokens = baiduResponse.Usage.PromptTokens
+			usage.CompletionTokens = baiduResponse.Usage.TotalTokens - baiduResponse.Usage.PromptTokens
+		}
+		response := streamResponseBaidu2OpenAI(&baiduResponse)
+		err = render.ObjectData(c, response)
+		if err != nil {
+			logger.SysError(err.Error())
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	render.Done(c)
+
 	err := resp.Body.Close()
 	if err != nil {
 		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil

relay/adaptor/cloudflare/adaptor.go

@@ -5,11 +5,13 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/relay/adaptor"
 	"github.com/songquanpeng/one-api/relay/meta"
 	"github.com/songquanpeng/one-api/relay/model"
+	"github.com/songquanpeng/one-api/relay/relaymode"
 )
 
 type Adaptor struct {
@@ -27,8 +29,33 @@ func (a *Adaptor) Init(meta *meta.Meta) {
 	a.meta = meta
 }
 
+// WorkerAI cannot be used across accounts with AIGateWay
+// https://developers.cloudflare.com/ai-gateway/providers/workersai/#openai-compatible-endpoints
+// https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/workers-ai
+func (a *Adaptor) isAIGateWay(baseURL string) bool {
+	return strings.HasPrefix(baseURL, "https://gateway.ai.cloudflare.com") && strings.HasSuffix(baseURL, "/workers-ai")
+}
+
 func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
-	return fmt.Sprintf("%s/client/v4/accounts/%s/ai/run/%s", meta.BaseURL, meta.Config.UserID, meta.ActualModelName), nil
+	isAIGateWay := a.isAIGateWay(meta.BaseURL)
+	var urlPrefix string
+	if isAIGateWay {
+		urlPrefix = meta.BaseURL
+	} else {
+		urlPrefix = fmt.Sprintf("%s/client/v4/accounts/%s/ai", meta.BaseURL, meta.Config.UserID)
+	}
+
+	switch meta.Mode {
+	case relaymode.ChatCompletions:
+		return fmt.Sprintf("%s/v1/chat/completions", urlPrefix), nil
+	case relaymode.Embeddings:
+		return fmt.Sprintf("%s/v1/embeddings", urlPrefix), nil
+	default:
+		if isAIGateWay {
+			return fmt.Sprintf("%s/%s", urlPrefix, meta.ActualModelName), nil
+		}
+		return fmt.Sprintf("%s/run/%s", urlPrefix, meta.ActualModelName), nil
+	}
 }
 
 func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *meta.Meta) error {
@@ -41,7 +68,14 @@ func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.G
 	if request == nil {
 		return nil, errors.New("request is nil")
 	}
-	return ConvertRequest(*request), nil
+	switch relayMode {
+	case relaymode.Completions:
+		return ConvertCompletionsRequest(*request), nil
+	case relaymode.ChatCompletions, relaymode.Embeddings:
+		return request, nil
+	default:
+		return nil, errors.New("not implemented")
+	}
 }
 
 func (a *Adaptor) DoRequest(c *gin.Context, meta *meta.Meta, requestBody io.Reader) (*http.Response, error) {

relay/adaptor/cloudflare/constant.go

@@ -1,6 +1,7 @@
 package cloudflare
 
 var ModelList = []string{
+	"@cf/meta/llama-3.1-8b-instruct",
 	"@cf/meta/llama-2-7b-chat-fp16",
 	"@cf/meta/llama-2-7b-chat-int8",
 	"@cf/mistral/mistral-7b-instruct-v0.1",

relay/adaptor/cloudflare/main.go

@@ -2,12 +2,14 @@ package cloudflare
 
 import (
 	"bufio"
-	"bytes"
 	"encoding/json"
 	"io"
 	"net/http"
 	"strings"
 
+	"github.com/songquanpeng/one-api/common/ctxkey"
+	"github.com/songquanpeng/one-api/common/render"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common"
 	"github.com/songquanpeng/one-api/common/helper"
@@ -16,114 +18,66 @@ import (
 	"github.com/songquanpeng/one-api/relay/model"
 )
 
-func ConvertRequest(textRequest model.GeneralOpenAIRequest) *Request {
-    var promptBuilder strings.Builder
-    for _, message := range textRequest.Messages {
-        promptBuilder.WriteString(message.StringContent())
-        promptBuilder.WriteString("\n")  // 添加换行符来分隔每个消息
-    }
-
-    return &Request{
-        MaxTokens:   textRequest.MaxTokens,
-        Prompt:      promptBuilder.String(),
-        Stream:      textRequest.Stream,
-        Temperature: textRequest.Temperature,
-    }
-}
-
-
-func ResponseCloudflare2OpenAI(cloudflareResponse *Response) *openai.TextResponse {
-	choice := openai.TextResponseChoice{
-		Index: 0,
-		Message: model.Message{
-			Role:    "assistant",
-			Content: cloudflareResponse.Result.Response,
-		},
-		FinishReason: "stop",
+func ConvertCompletionsRequest(textRequest model.GeneralOpenAIRequest) *Request {
+	p, _ := textRequest.Prompt.(string)
+	return &Request{
+		Prompt:      p,
+		MaxTokens:   textRequest.MaxTokens,
+		Stream:      textRequest.Stream,
+		Temperature: textRequest.Temperature,
 	}
-	fullTextResponse := openai.TextResponse{
-		Object:  "chat.completion",
-		Created: helper.GetTimestamp(),
-		Choices: []openai.TextResponseChoice{choice},
-	}
-	return &fullTextResponse
-}
-
-func StreamResponseCloudflare2OpenAI(cloudflareResponse *StreamResponse) *openai.ChatCompletionsStreamResponse {
-	var choice openai.ChatCompletionsStreamResponseChoice
-	choice.Delta.Content = cloudflareResponse.Response
-	choice.Delta.Role = "assistant"
-	openaiResponse := openai.ChatCompletionsStreamResponse{
-		Object:  "chat.completion.chunk",
-		Choices: []openai.ChatCompletionsStreamResponseChoice{choice},
-		Created: helper.GetTimestamp(),
-	}
-	return &openaiResponse
 }
 
 func StreamHandler(c *gin.Context, resp *http.Response, promptTokens int, modelName string) (*model.ErrorWithStatusCode, *model.Usage) {
 	scanner := bufio.NewScanner(resp.Body)
-	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
-		if atEOF && len(data) == 0 {
-			return 0, nil, nil
-		}
-		if i := bytes.IndexByte(data, '\n'); i >= 0 {
-			return i + 1, data[0:i], nil
-		}
-		if atEOF {
-			return len(data), data, nil
-		}
-		return 0, nil, nil
-	})
+	scanner.Split(bufio.ScanLines)
 
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
-	go func() {
-		for scanner.Scan() {
-			data := scanner.Text()
-			if len(data) < len("data: ") {
-				continue
-			}
-			data = strings.TrimPrefix(data, "data: ")
-			dataChan <- data
-		}
-		stopChan <- true
-	}()
 	common.SetEventStreamHeaders(c)
 	id := helper.GetResponseID(c)
-	responseModel := c.GetString("original_model")
+	responseModel := c.GetString(ctxkey.OriginalModel)
 	var responseText string
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			// some implementations may add \r at the end of data
-			data = strings.TrimSuffix(data, "\r")
-			var cloudflareResponse StreamResponse
-			err := json.Unmarshal([]byte(data), &cloudflareResponse)
-			if err != nil {
-				logger.SysError("error unmarshalling stream response: " + err.Error())
-				return true
-			}
-			response := StreamResponseCloudflare2OpenAI(&cloudflareResponse)
-			if response == nil {
-				return true
-			}
-			responseText += cloudflareResponse.Response
-			response.Id = id
-			response.Model = responseModel
-			jsonStr, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonStr)})
-			return true
-		case <-stopChan:
-			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
-			return false
+
+	for scanner.Scan() {
+		data := scanner.Text()
+		if len(data) < len("data: ") {
+			continue
 		}
-	})
-	_ = resp.Body.Close()
+		data = strings.TrimPrefix(data, "data: ")
+		data = strings.TrimSuffix(data, "\r")
+
+		if data == "[DONE]" {
+			break
+		}
+
+		var response openai.ChatCompletionsStreamResponse
+		err := json.Unmarshal([]byte(data), &response)
+		if err != nil {
+			logger.SysError("error unmarshalling stream response: " + err.Error())
+			continue
+		}
+		for _, v := range response.Choices {
+			v.Delta.Role = "assistant"
+			responseText += v.Delta.StringContent()
+		}
+		response.Id = id
+		response.Model = modelName
+		err = render.ObjectData(c, response)
+		if err != nil {
+			logger.SysError(err.Error())
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	render.Done(c)
+
+	err := resp.Body.Close()
+	if err != nil {
+		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil
+	}
+
 	usage := openai.ResponseText2Usage(responseText, responseModel, promptTokens)
 	return nil, usage
 }
@@ -137,22 +91,25 @@ func Handler(c *gin.Context, resp *http.Response, promptTokens int, modelName st
 	if err != nil {
 		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil
 	}
-	var cloudflareResponse Response
-	err = json.Unmarshal(responseBody, &cloudflareResponse)
+	var response openai.TextResponse
+	err = json.Unmarshal(responseBody, &response)
 	if err != nil {
 		return openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError), nil
 	}
-	fullTextResponse := ResponseCloudflare2OpenAI(&cloudflareResponse)
-	fullTextResponse.Model = modelName
-	usage := openai.ResponseText2Usage(cloudflareResponse.Result.Response, modelName, promptTokens)
-	fullTextResponse.Usage = *usage
-	fullTextResponse.Id = helper.GetResponseID(c)
-	jsonResponse, err := json.Marshal(fullTextResponse)
+	response.Model = modelName
+	var responseText string
+	for _, v := range response.Choices {
+		responseText += v.Message.Content.(string)
+	}
+	usage := openai.ResponseText2Usage(responseText, modelName, promptTokens)
+	response.Usage = *usage
+	response.Id = helper.GetResponseID(c)
+	jsonResponse, err := json.Marshal(response)
 	if err != nil {
 		return openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError), nil
 	}
 	c.Writer.Header().Set("Content-Type", "application/json")
 	c.Writer.WriteHeader(resp.StatusCode)
-	_, err = c.Writer.Write(jsonResponse)
+	_, _ = c.Writer.Write(jsonResponse)
 	return nil, usage
 }

relay/adaptor/cloudflare/model.go

@@ -1,25 +1,13 @@
 package cloudflare
 
+import "github.com/songquanpeng/one-api/relay/model"
+
 type Request struct {
-	Lora        string  `json:"lora,omitempty"`
-	MaxTokens   int     `json:"max_tokens,omitempty"`
-	Prompt      string  `json:"prompt,omitempty"`
-	Raw         bool    `json:"raw,omitempty"`
-	Stream      bool    `json:"stream,omitempty"`
-	Temperature float64 `json:"temperature,omitempty"`
-}
-
-type Result struct {
-	Response string `json:"response"`
-}
-
-type Response struct {
-	Result   Result   `json:"result"`
-	Success  bool     `json:"success"`
-	Errors   []string `json:"errors"`
-	Messages []string `json:"messages"`
-}
-
-type StreamResponse struct {
-	Response string `json:"response"`
+	Messages    []model.Message `json:"messages,omitempty"`
+	Lora        string          `json:"lora,omitempty"`
+	MaxTokens   int             `json:"max_tokens,omitempty"`
+	Prompt      string          `json:"prompt,omitempty"`
+	Raw         bool            `json:"raw,omitempty"`
+	Stream      bool            `json:"stream,omitempty"`
+	Temperature float64         `json:"temperature,omitempty"`
 }

relay/adaptor/cohere/main.go

@@ -2,9 +2,9 @@ package cohere
 
 import (
 	"bufio"
-	"bytes"
 	"encoding/json"
 	"fmt"
+	"github.com/songquanpeng/one-api/common/render"
 	"io"
 	"net/http"
 	"strings"
@@ -134,66 +134,53 @@ func ResponseCohere2OpenAI(cohereResponse *Response) *openai.TextResponse {
 func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusCode, *model.Usage) {
 	createdTime := helper.GetTimestamp()
 	scanner := bufio.NewScanner(resp.Body)
-	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
-		if atEOF && len(data) == 0 {
-			return 0, nil, nil
-		}
-		if i := bytes.IndexByte(data, '\n'); i >= 0 {
-			return i + 1, data[0:i], nil
-		}
-		if atEOF {
-			return len(data), data, nil
-		}
-		return 0, nil, nil
-	})
+	scanner.Split(bufio.ScanLines)
 
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
-	go func() {
-		for scanner.Scan() {
-			data := scanner.Text()
-			dataChan <- data
-		}
-		stopChan <- true
-	}()
 	common.SetEventStreamHeaders(c)
 	var usage model.Usage
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			// some implementations may add \r at the end of data
-			data = strings.TrimSuffix(data, "\r")
-			var cohereResponse StreamResponse
-			err := json.Unmarshal([]byte(data), &cohereResponse)
-			if err != nil {
-				logger.SysError("error unmarshalling stream response: " + err.Error())
-				return true
-			}
-			response, meta := StreamResponseCohere2OpenAI(&cohereResponse)
-			if meta != nil {
-				usage.PromptTokens += meta.Meta.Tokens.InputTokens
-				usage.CompletionTokens += meta.Meta.Tokens.OutputTokens
-				return true
-			}
-			if response == nil {
-				return true
-			}
-			response.Id = fmt.Sprintf("chatcmpl-%d", createdTime)
-			response.Model = c.GetString("original_model")
-			response.Created = createdTime
-			jsonStr, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonStr)})
-			return true
-		case <-stopChan:
-			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
-			return false
+
+	for scanner.Scan() {
+		data := scanner.Text()
+		data = strings.TrimSuffix(data, "\r")
+
+		var cohereResponse StreamResponse
+		err := json.Unmarshal([]byte(data), &cohereResponse)
+		if err != nil {
+			logger.SysError("error unmarshalling stream response: " + err.Error())
+			continue
 		}
-	})
-	_ = resp.Body.Close()
+
+		response, meta := StreamResponseCohere2OpenAI(&cohereResponse)
+		if meta != nil {
+			usage.PromptTokens += meta.Meta.Tokens.InputTokens
+			usage.CompletionTokens += meta.Meta.Tokens.OutputTokens
+			continue
+		}
+		if response == nil {
+			continue
+		}
+
+		response.Id = fmt.Sprintf("chatcmpl-%d", createdTime)
+		response.Model = c.GetString("original_model")
+		response.Created = createdTime
+
+		err = render.ObjectData(c, response)
+		if err != nil {
+			logger.SysError(err.Error())
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	render.Done(c)
+
+	err := resp.Body.Close()
+	if err != nil {
+		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil
+	}
+
 	return nil, &usage
 }
 

relay/adaptor/coze/main.go

@@ -4,6 +4,11 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
+	"github.com/songquanpeng/one-api/common/render"
+	"io"
+	"net/http"
+	"strings"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common"
 	"github.com/songquanpeng/one-api/common/conv"
@@ -12,9 +17,6 @@ import (
 	"github.com/songquanpeng/one-api/relay/adaptor/coze/constant/messagetype"
 	"github.com/songquanpeng/one-api/relay/adaptor/openai"
 	"github.com/songquanpeng/one-api/relay/model"
-	"io"
-	"net/http"
-	"strings"
 )
 
 // https://www.coze.com/open
@@ -109,69 +111,54 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 	var responseText string
 	createdTime := helper.GetTimestamp()
 	scanner := bufio.NewScanner(resp.Body)
-	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
-		if atEOF && len(data) == 0 {
-			return 0, nil, nil
-		}
-		if i := strings.Index(string(data), "\n"); i >= 0 {
-			return i + 1, data[0:i], nil
-		}
-		if atEOF {
-			return len(data), data, nil
-		}
-		return 0, nil, nil
-	})
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
-	go func() {
-		for scanner.Scan() {
-			data := scanner.Text()
-			if len(data) < 5 {
-				continue
-			}
-			if !strings.HasPrefix(data, "data:") {
-				continue
-			}
-			data = strings.TrimPrefix(data, "data:")
-			dataChan <- data
-		}
-		stopChan <- true
-	}()
+	scanner.Split(bufio.ScanLines)
+
 	common.SetEventStreamHeaders(c)
 	var modelName string
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			// some implementations may add \r at the end of data
-			data = strings.TrimSuffix(data, "\r")
-			var cozeResponse StreamResponse
-			err := json.Unmarshal([]byte(data), &cozeResponse)
-			if err != nil {
-				logger.SysError("error unmarshalling stream response: " + err.Error())
-				return true
-			}
-			response, _ := StreamResponseCoze2OpenAI(&cozeResponse)
-			if response == nil {
-				return true
-			}
-			for _, choice := range response.Choices {
-				responseText += conv.AsString(choice.Delta.Content)
-			}
-			response.Model = modelName
-			response.Created = createdTime
-			jsonStr, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonStr)})
-			return true
-		case <-stopChan:
-			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
-			return false
+
+	for scanner.Scan() {
+		data := scanner.Text()
+		if len(data) < 5 || !strings.HasPrefix(data, "data:") {
+			continue
 		}
-	})
-	_ = resp.Body.Close()
+		data = strings.TrimPrefix(data, "data:")
+		data = strings.TrimSuffix(data, "\r")
+
+		var cozeResponse StreamResponse
+		err := json.Unmarshal([]byte(data), &cozeResponse)
+		if err != nil {
+			logger.SysError("error unmarshalling stream response: " + err.Error())
+			continue
+		}
+
+		response, _ := StreamResponseCoze2OpenAI(&cozeResponse)
+		if response == nil {
+			continue
+		}
+
+		for _, choice := range response.Choices {
+			responseText += conv.AsString(choice.Delta.Content)
+		}
+		response.Model = modelName
+		response.Created = createdTime
+
+		err = render.ObjectData(c, response)
+		if err != nil {
+			logger.SysError(err.Error())
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	render.Done(c)
+
+	err := resp.Body.Close()
+	if err != nil {
+		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil
+	}
+
 	return nil, &responseText
 }
 

relay/adaptor/doubao/main.go

@@ -7,8 +7,12 @@ import (
 )
 
 func GetRequestURL(meta *meta.Meta) (string, error) {
-	if meta.Mode == relaymode.ChatCompletions {
+	switch meta.Mode {
+	case relaymode.ChatCompletions:
 		return fmt.Sprintf("%s/api/v3/chat/completions", meta.BaseURL), nil
+	case relaymode.Embeddings:
+		return fmt.Sprintf("%s/api/v3/embeddings", meta.BaseURL), nil
+	default:
 	}
 	return "", fmt.Errorf("unsupported relay mode %d for doubao", meta.Mode)
 }

relay/adaptor/gemini/constants.go

@@ -3,6 +3,5 @@ package gemini
 // https://ai.google.dev/models/gemini
 
 var ModelList = []string{
-	"gemini-pro", "gemini-1.0-pro-001", "gemini-1.5-pro",
-	"gemini-pro-vision", "gemini-1.0-pro-vision-001", "embedding-001", "text-embedding-004",
+	"gemini-pro", "gemini-1.0-pro", "gemini-1.5-flash", "gemini-1.5-pro", "text-embedding-004", "aqa",
 }

relay/adaptor/gemini/main.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
+	"github.com/songquanpeng/one-api/common/render"
 	"io"
 	"net/http"
 	"strings"
@@ -245,8 +246,10 @@ func responseGeminiChat2OpenAI(response *ChatResponse) *openai.TextResponse {
 func streamResponseGeminiChat2OpenAI(geminiResponse *ChatResponse) *openai.ChatCompletionsStreamResponse {
 	var choice openai.ChatCompletionsStreamResponseChoice
 	choice.Delta.Content = geminiResponse.GetResponseText()
-	choice.FinishReason = &constant.StopFinishReason
+	//choice.FinishReason = &constant.StopFinishReason
 	var response openai.ChatCompletionsStreamResponse
+	response.Id = fmt.Sprintf("chatcmpl-%s", random.GetUUID())
+	response.Created = helper.GetTimestamp()
 	response.Object = "chat.completion.chunk"
 	response.Model = "gemini"
 	response.Choices = []openai.ChatCompletionsStreamResponseChoice{choice}
@@ -273,64 +276,50 @@ func embeddingResponseGemini2OpenAI(response *EmbeddingResponse) *openai.Embeddi
 func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusCode, string) {
 	responseText := ""
 	scanner := bufio.NewScanner(resp.Body)
-	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
-		if atEOF && len(data) == 0 {
-			return 0, nil, nil
-		}
-		if i := strings.Index(string(data), "\n"); i >= 0 {
-			return i + 1, data[0:i], nil
-		}
-		if atEOF {
-			return len(data), data, nil
-		}
-		return 0, nil, nil
-	})
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
-	go func() {
-		for scanner.Scan() {
-			data := scanner.Text()
-			data = strings.TrimSpace(data)
-			if !strings.HasPrefix(data, "data: ") {
-				continue
-			}
-			data = strings.TrimPrefix(data, "data: ")
-			data = strings.TrimSuffix(data, "\"")
-			dataChan <- data
-		}
-		stopChan <- true
-	}()
+	scanner.Split(bufio.ScanLines)
+
 	common.SetEventStreamHeaders(c)
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			var geminiResponse ChatResponse
-			err := json.Unmarshal([]byte(data), &geminiResponse)
-			if err != nil {
-				logger.SysError("error unmarshalling stream response: " + err.Error())
-				return true
-			}
-			response := streamResponseGeminiChat2OpenAI(&geminiResponse)
-			if response == nil {
-				return true
-			}
-			responseText += response.Choices[0].Delta.StringContent()
-			jsonResponse, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonResponse)})
-			return true
-		case <-stopChan:
-			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
-			return false
+
+	for scanner.Scan() {
+		data := scanner.Text()
+		data = strings.TrimSpace(data)
+		if !strings.HasPrefix(data, "data: ") {
+			continue
 		}
-	})
+		data = strings.TrimPrefix(data, "data: ")
+		data = strings.TrimSuffix(data, "\"")
+
+		var geminiResponse ChatResponse
+		err := json.Unmarshal([]byte(data), &geminiResponse)
+		if err != nil {
+			logger.SysError("error unmarshalling stream response: " + err.Error())
+			continue
+		}
+
+		response := streamResponseGeminiChat2OpenAI(&geminiResponse)
+		if response == nil {
+			continue
+		}
+
+		responseText += response.Choices[0].Delta.StringContent()
+
+		err = render.ObjectData(c, response)
+		if err != nil {
+			logger.SysError(err.Error())
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	render.Done(c)
+
 	err := resp.Body.Close()
 	if err != nil {
 		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), ""
 	}
+
 	return nil, responseText
 }
 

relay/adaptor/groq/constants.go

@@ -4,9 +4,14 @@ package groq
 
 var ModelList = []string{
 	"gemma-7b-it",
-	"llama2-7b-2048",
-	"llama2-70b-4096",
 	"mixtral-8x7b-32768",
 	"llama3-8b-8192",
 	"llama3-70b-8192",
+	"gemma2-9b-it",
+	"llama-3.1-405b-reasoning",
+	"llama-3.1-70b-versatile",
+	"llama-3.1-8b-instant",
+	"llama3-groq-70b-8192-tool-use-preview",
+	"llama3-groq-8b-8192-tool-use-preview",
+	"whisper-large-v3",
 }

relay/adaptor/novita/constants.go

@@ -0,0 +1,19 @@
+package novita
+
+// https://novita.ai/llm-api
+
+var ModelList = []string{
+	"meta-llama/llama-3-8b-instruct",
+	"meta-llama/llama-3-70b-instruct",
+	"nousresearch/hermes-2-pro-llama-3-8b",
+	"nousresearch/nous-hermes-llama2-13b",
+	"mistralai/mistral-7b-instruct",
+	"cognitivecomputations/dolphin-mixtral-8x22b",
+	"sao10k/l3-70b-euryale-v2.1",
+	"sophosympatheia/midnight-rose-70b",
+	"gryphe/mythomax-l2-13b",
+	"Nous-Hermes-2-Mixtral-8x7B-DPO",
+	"lzlv_70b",
+	"teknium/openhermes-2.5-mistral-7b",
+	"microsoft/wizardlm-2-8x22b",
+}

relay/adaptor/novita/main.go

@@ -0,0 +1,15 @@
+package novita
+
+import (
+	"fmt"
+
+	"github.com/songquanpeng/one-api/relay/meta"
+	"github.com/songquanpeng/one-api/relay/relaymode"
+)
+
+func GetRequestURL(meta *meta.Meta) (string, error) {
+	if meta.Mode == relaymode.ChatCompletions {
+		return fmt.Sprintf("%s/chat/completions", meta.BaseURL), nil
+	}
+	return "", fmt.Errorf("unsupported relay mode %d for novita", meta.Mode)
+}

relay/adaptor/ollama/adaptor.go

@@ -24,7 +24,7 @@ func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
 	// https://github.com/ollama/ollama/blob/main/docs/api.md
 	fullRequestURL := fmt.Sprintf("%s/api/chat", meta.BaseURL)
 	if meta.Mode == relaymode.Embeddings {
-		fullRequestURL = fmt.Sprintf("%s/api/embeddings", meta.BaseURL)
+		fullRequestURL = fmt.Sprintf("%s/api/embed", meta.BaseURL)
 	}
 	return fullRequestURL, nil
 }

relay/adaptor/ollama/main.go

@@ -5,12 +5,14 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/songquanpeng/one-api/common/helper"
-	"github.com/songquanpeng/one-api/common/random"
+	"github.com/songquanpeng/one-api/common/render"
 	"io"
 	"net/http"
 	"strings"
 
+	"github.com/songquanpeng/one-api/common/helper"
+	"github.com/songquanpeng/one-api/common/random"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common"
 	"github.com/songquanpeng/one-api/common/image"
@@ -29,6 +31,8 @@ func ConvertRequest(request model.GeneralOpenAIRequest) *ChatRequest {
 			TopP:             request.TopP,
 			FrequencyPenalty: request.FrequencyPenalty,
 			PresencePenalty:  request.PresencePenalty,
+			NumPredict:  	  request.MaxTokens,
+			NumCtx:  	  request.NumCtx,
 		},
 		Stream: request.Stream,
 	}
@@ -105,61 +109,67 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 			return 0, nil, nil
 		}
 		if i := strings.Index(string(data), "}\n"); i >= 0 {
-			return i + 2, data[0:i], nil
+			return i + 2, data[0 : i+1], nil
 		}
 		if atEOF {
 			return len(data), data, nil
 		}
 		return 0, nil, nil
 	})
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
-	go func() {
-		for scanner.Scan() {
-			data := strings.TrimPrefix(scanner.Text(), "}")
-			dataChan <- data + "}"
-		}
-		stopChan <- true
-	}()
+
 	common.SetEventStreamHeaders(c)
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			var ollamaResponse ChatResponse
-			err := json.Unmarshal([]byte(data), &ollamaResponse)
-			if err != nil {
-				logger.SysError("error unmarshalling stream response: " + err.Error())
-				return true
-			}
-			if ollamaResponse.EvalCount != 0 {
-				usage.PromptTokens = ollamaResponse.PromptEvalCount
-				usage.CompletionTokens = ollamaResponse.EvalCount
-				usage.TotalTokens = ollamaResponse.PromptEvalCount + ollamaResponse.EvalCount
-			}
-			response := streamResponseOllama2OpenAI(&ollamaResponse)
-			jsonResponse, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonResponse)})
-			return true
-		case <-stopChan:
-			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
-			return false
+
+	for scanner.Scan() {
+		data := scanner.Text()
+		if strings.HasPrefix(data, "}") {
+		    data = strings.TrimPrefix(data, "}") + "}"
 		}
-	})
+
+		var ollamaResponse ChatResponse
+		err := json.Unmarshal([]byte(data), &ollamaResponse)
+		if err != nil {
+			logger.SysError("error unmarshalling stream response: " + err.Error())
+			continue
+		}
+
+		if ollamaResponse.EvalCount != 0 {
+			usage.PromptTokens = ollamaResponse.PromptEvalCount
+			usage.CompletionTokens = ollamaResponse.EvalCount
+			usage.TotalTokens = ollamaResponse.PromptEvalCount + ollamaResponse.EvalCount
+		}
+
+		response := streamResponseOllama2OpenAI(&ollamaResponse)
+		err = render.ObjectData(c, response)
+		if err != nil {
+			logger.SysError(err.Error())
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	render.Done(c)
+
 	err := resp.Body.Close()
 	if err != nil {
 		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil
 	}
+
 	return nil, &usage
 }
 
 func ConvertEmbeddingRequest(request model.GeneralOpenAIRequest) *EmbeddingRequest {
 	return &EmbeddingRequest{
-		Model:  request.Model,
-		Prompt: strings.Join(request.ParseInput(), " "),
+		Model: request.Model,
+		Input: request.ParseInput(),
+		Options: &Options{
+			Seed:             int(request.Seed),
+			Temperature:      request.Temperature,
+			TopP:             request.TopP,
+			FrequencyPenalty: request.FrequencyPenalty,
+			PresencePenalty:  request.PresencePenalty,
+		},
 	}
 }
 
@@ -202,15 +212,17 @@ func embeddingResponseOllama2OpenAI(response *EmbeddingResponse) *openai.Embeddi
 	openAIEmbeddingResponse := openai.EmbeddingResponse{
 		Object: "list",
 		Data:   make([]openai.EmbeddingResponseItem, 0, 1),
-		Model:  "text-embedding-v1",
+		Model:  response.Model,
 		Usage:  model.Usage{TotalTokens: 0},
 	}
 
-	openAIEmbeddingResponse.Data = append(openAIEmbeddingResponse.Data, openai.EmbeddingResponseItem{
-		Object:    `embedding`,
-		Index:     0,
-		Embedding: response.Embedding,
-	})
+	for i, embedding := range response.Embeddings {
+		openAIEmbeddingResponse.Data = append(openAIEmbeddingResponse.Data, openai.EmbeddingResponseItem{
+			Object:    `embedding`,
+			Index:     i,
+			Embedding: embedding,
+		})
+	}
 	return &openAIEmbeddingResponse
 }
 

relay/adaptor/ollama/model.go

@@ -7,6 +7,8 @@ type Options struct {
 	TopP             float64 `json:"top_p,omitempty"`
 	FrequencyPenalty float64 `json:"frequency_penalty,omitempty"`
 	PresencePenalty  float64 `json:"presence_penalty,omitempty"`
+	NumPredict  	 int 	 `json:"num_predict,omitempty"`
+	NumCtx  	 int 	 `json:"num_ctx,omitempty"`
 }
 
 type Message struct {
@@ -37,11 +39,15 @@ type ChatResponse struct {
 }
 
 type EmbeddingRequest struct {
-	Model  string `json:"model"`
-	Prompt string `json:"prompt"`
+	Model string   `json:"model"`
+	Input []string `json:"input"`
+	// Truncate  bool     `json:"truncate,omitempty"`
+	Options *Options `json:"options,omitempty"`
+	// KeepAlive string   `json:"keep_alive,omitempty"`
 }
 
 type EmbeddingResponse struct {
-	Error     string    `json:"error,omitempty"`
-	Embedding []float64 `json:"embedding,omitempty"`
+	Error      string      `json:"error,omitempty"`
+	Model      string      `json:"model"`
+	Embeddings [][]float64 `json:"embeddings"`
 }

relay/adaptor/openai/adaptor.go

@@ -3,17 +3,19 @@ package openai
 import (
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/relay/adaptor"
 	"github.com/songquanpeng/one-api/relay/adaptor/doubao"
 	"github.com/songquanpeng/one-api/relay/adaptor/minimax"
+	"github.com/songquanpeng/one-api/relay/adaptor/novita"
 	"github.com/songquanpeng/one-api/relay/channeltype"
 	"github.com/songquanpeng/one-api/relay/meta"
 	"github.com/songquanpeng/one-api/relay/model"
 	"github.com/songquanpeng/one-api/relay/relaymode"
-	"io"
-	"net/http"
-	"strings"
 )
 
 type Adaptor struct {
@@ -53,6 +55,8 @@ func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
 			return GetFullRequestURL("https://apivip.aiproxy.io", meta.RequestURLPath, meta.ChannelType), nil
 		}
 		fallthrough
+	case channeltype.Novita:
+		return novita.GetRequestURL(meta)
 	default:
 		return GetFullRequestURL(meta.BaseURL, meta.RequestURLPath, meta.ChannelType), nil
 	}

relay/adaptor/openai/compatible.go

@@ -10,8 +10,10 @@ import (
 	"github.com/songquanpeng/one-api/relay/adaptor/minimax"
 	"github.com/songquanpeng/one-api/relay/adaptor/mistral"
 	"github.com/songquanpeng/one-api/relay/adaptor/moonshot"
+	"github.com/songquanpeng/one-api/relay/adaptor/novita"
 	"github.com/songquanpeng/one-api/relay/adaptor/stepfun"
 	"github.com/songquanpeng/one-api/relay/adaptor/togetherai"
+	"github.com/songquanpeng/one-api/relay/adaptor/siliconflow"
 	"github.com/songquanpeng/one-api/relay/channeltype"
 )
 
@@ -28,6 +30,8 @@ var CompatibleChannels = []int{
 	channeltype.StepFun,
 	channeltype.DeepSeek,
 	channeltype.TogetherAI,
+	channeltype.Novita,
+	channeltype.SiliconFlow,
 }
 
 func GetCompatibleChannelMeta(channelType int) (string, []string) {
@@ -56,6 +60,10 @@ func GetCompatibleChannelMeta(channelType int) (string, []string) {
 		return "together.ai", togetherai.ModelList
 	case channeltype.Doubao:
 		return "doubao", doubao.ModelList
+	case channeltype.Novita:
+		return "novita", novita.ModelList
+	case channeltype.SiliconFlow:
+		return "siliconflow", siliconflow.ModelList
 	default:
 		return "openai", ModelList
 	}

relay/adaptor/openai/constants.go

@@ -8,6 +8,7 @@ var ModelList = []string{
 	"gpt-4-32k", "gpt-4-32k-0314", "gpt-4-32k-0613",
 	"gpt-4-turbo-preview", "gpt-4-turbo", "gpt-4-turbo-2024-04-09",
 	"gpt-4o", "gpt-4o-2024-05-13",
+	"gpt-4o-mini", "gpt-4o-mini-2024-07-18",
 	"gpt-4-vision-preview",
 	"text-embedding-ada-002", "text-embedding-3-small", "text-embedding-3-large",
 	"text-curie-001", "text-babbage-001", "text-ada-001", "text-davinci-002", "text-davinci-003",

relay/adaptor/openai/main.go

@@ -4,15 +4,18 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/songquanpeng/one-api/common/render"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common"
 	"github.com/songquanpeng/one-api/common/conv"
 	"github.com/songquanpeng/one-api/common/logger"
 	"github.com/songquanpeng/one-api/relay/model"
 	"github.com/songquanpeng/one-api/relay/relaymode"
-	"io"
-	"net/http"
-	"strings"
 )
 
 const (
@@ -24,88 +27,72 @@ const (
 func StreamHandler(c *gin.Context, resp *http.Response, relayMode int) (*model.ErrorWithStatusCode, string, *model.Usage) {
 	responseText := ""
 	scanner := bufio.NewScanner(resp.Body)
-	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
-		if atEOF && len(data) == 0 {
-			return 0, nil, nil
-		}
-		if i := strings.Index(string(data), "\n"); i >= 0 {
-			return i + 1, data[0:i], nil
-		}
-		if atEOF {
-			return len(data), data, nil
-		}
-		return 0, nil, nil
-	})
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
+	scanner.Split(bufio.ScanLines)
 	var usage *model.Usage
-	go func() {
-		for scanner.Scan() {
-			data := scanner.Text()
-			if len(data) < dataPrefixLength { // ignore blank line or wrong format
-				continue
-			}
-			if data[:dataPrefixLength] != dataPrefix && data[:dataPrefixLength] != done {
-				continue
-			}
-			if strings.HasPrefix(data[dataPrefixLength:], done) {
-				dataChan <- data
-				continue
-			}
-			switch relayMode {
-			case relaymode.ChatCompletions:
-				var streamResponse ChatCompletionsStreamResponse
-				err := json.Unmarshal([]byte(data[dataPrefixLength:]), &streamResponse)
-				if err != nil {
-					logger.SysError("error unmarshalling stream response: " + err.Error())
-					dataChan <- data // if error happened, pass the data to client
-					continue         // just ignore the error
-				}
-				if len(streamResponse.Choices) == 0 {
-					// but for empty choice, we should not pass it to client, this is for azure
-					continue // just ignore empty choice
-				}
-				dataChan <- data
-				for _, choice := range streamResponse.Choices {
-					responseText += conv.AsString(choice.Delta.Content)
-				}
-				if streamResponse.Usage != nil {
-					usage = streamResponse.Usage
-				}
-			case relaymode.Completions:
-				dataChan <- data
-				var streamResponse CompletionsStreamResponse
-				err := json.Unmarshal([]byte(data[dataPrefixLength:]), &streamResponse)
-				if err != nil {
-					logger.SysError("error unmarshalling stream response: " + err.Error())
-					continue
-				}
-				for _, choice := range streamResponse.Choices {
-					responseText += choice.Text
-				}
-			}
-		}
-		stopChan <- true
-	}()
+
 	common.SetEventStreamHeaders(c)
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			if strings.HasPrefix(data, "data: [DONE]") {
-				data = data[:12]
-			}
-			// some implementations may add \r at the end of data
-			data = strings.TrimSuffix(data, "\r")
-			c.Render(-1, common.CustomEvent{Data: data})
-			return true
-		case <-stopChan:
-			return false
+
+	doneRendered := false
+	for scanner.Scan() {
+		data := scanner.Text()
+		if len(data) < dataPrefixLength { // ignore blank line or wrong format
+			continue
 		}
-	})
+		if data[:dataPrefixLength] != dataPrefix && data[:dataPrefixLength] != done {
+			continue
+		}
+		if strings.HasPrefix(data[dataPrefixLength:], done) {
+			render.StringData(c, data)
+			doneRendered = true
+			continue
+		}
+		switch relayMode {
+		case relaymode.ChatCompletions:
+			var streamResponse ChatCompletionsStreamResponse
+			err := json.Unmarshal([]byte(data[dataPrefixLength:]), &streamResponse)
+			if err != nil {
+				logger.SysError("error unmarshalling stream response: " + err.Error())
+				render.StringData(c, data) // if error happened, pass the data to client
+				continue                   // just ignore the error
+			}
+			if len(streamResponse.Choices) == 0 {
+				// but for empty choice, we should not pass it to client, this is for azure
+				continue // just ignore empty choice
+			}
+			render.StringData(c, data)
+			for _, choice := range streamResponse.Choices {
+				responseText += conv.AsString(choice.Delta.Content)
+			}
+			if streamResponse.Usage != nil {
+				usage = streamResponse.Usage
+			}
+		case relaymode.Completions:
+			render.StringData(c, data)
+			var streamResponse CompletionsStreamResponse
+			err := json.Unmarshal([]byte(data[dataPrefixLength:]), &streamResponse)
+			if err != nil {
+				logger.SysError("error unmarshalling stream response: " + err.Error())
+				continue
+			}
+			for _, choice := range streamResponse.Choices {
+				responseText += choice.Text
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	if !doneRendered {
+		render.Done(c)
+	}
+
 	err := resp.Body.Close()
 	if err != nil {
 		return ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), "", nil
 	}
+
 	return nil, responseText, usage
 }
 
@@ -149,7 +136,7 @@ func Handler(c *gin.Context, resp *http.Response, promptTokens int, modelName st
 		return ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil
 	}
 
-	if textResponse.Usage.TotalTokens == 0 {
+	if textResponse.Usage.TotalTokens == 0 || (textResponse.Usage.PromptTokens == 0 && textResponse.Usage.CompletionTokens == 0) {
 		completionTokens := 0
 		for _, choice := range textResponse.Choices {
 			completionTokens += CountTokenText(choice.Message.StringContent(), modelName)

relay/adaptor/openai/token.go

@@ -97,7 +97,11 @@ func CountTokenMessages(messages []model.Message, model string) int {
 				m := it.(map[string]any)
 				switch m["type"] {
 				case "text":
-					tokenNum += getTokenNum(tokenEncoder, m["text"].(string))
+					if textValue, ok := m["text"]; ok {
+						if textString, ok := textValue.(string); ok {
+							tokenNum += getTokenNum(tokenEncoder, textString)
+						}
+					}
 				case "image_url":
 					imageUrl, ok := m["image_url"].(map[string]any)
 					if ok {
@@ -106,7 +110,7 @@ func CountTokenMessages(messages []model.Message, model string) int {
 						if imageUrl["detail"] != nil {
 							detail = imageUrl["detail"].(string)
 						}
-						imageTokens, err := countImageTokens(url, detail)
+						imageTokens, err := countImageTokens(url, detail, model)
 						if err != nil {
 							logger.SysError("error counting image tokens: " + err.Error())
 						} else {
@@ -130,11 +134,15 @@ const (
 	lowDetailCost         = 85
 	highDetailCostPerTile = 170
 	additionalCost        = 85
+	// gpt-4o-mini cost higher than other model
+	gpt4oMiniLowDetailCost  = 2833
+	gpt4oMiniHighDetailCost = 5667
+	gpt4oMiniAdditionalCost = 2833
 )
 
 // https://platform.openai.com/docs/guides/vision/calculating-costs
 // https://github.com/openai/openai-cookbook/blob/05e3f9be4c7a2ae7ecf029a7c32065b024730ebe/examples/How_to_count_tokens_with_tiktoken.ipynb
-func countImageTokens(url string, detail string) (_ int, err error) {
+func countImageTokens(url string, detail string, model string) (_ int, err error) {
 	var fetchSize = true
 	var width, height int
 	// Reference: https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding
@@ -168,6 +176,9 @@ func countImageTokens(url string, detail string) (_ int, err error) {
 	}
 	switch detail {
 	case "low":
+		if strings.HasPrefix(model, "gpt-4o-mini") {
+			return gpt4oMiniLowDetailCost, nil
+		}
 		return lowDetailCost, nil
 	case "high":
 		if fetchSize {
@@ -187,6 +198,9 @@ func countImageTokens(url string, detail string) (_ int, err error) {
 			height = int(float64(height) * ratio)
 		}
 		numSquares := int(math.Ceil(float64(width)/512) * math.Ceil(float64(height)/512))
+		if strings.HasPrefix(model, "gpt-4o-mini") {
+			return numSquares*gpt4oMiniHighDetailCost + gpt4oMiniAdditionalCost, nil
+		}
 		result := numSquares*highDetailCostPerTile + additionalCost
 		return result, nil
 	default:

relay/adaptor/palm/palm.go

@@ -3,6 +3,10 @@ package palm
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/songquanpeng/one-api/common/render"
+	"io"
+	"net/http"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common"
 	"github.com/songquanpeng/one-api/common/helper"
@@ -11,8 +15,6 @@ import (
 	"github.com/songquanpeng/one-api/relay/adaptor/openai"
 	"github.com/songquanpeng/one-api/relay/constant"
 	"github.com/songquanpeng/one-api/relay/model"
-	"io"
-	"net/http"
 )
 
 // https://developers.generativeai.google/api/rest/generativelanguage/models/generateMessage#request-body
@@ -77,58 +79,51 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 	responseText := ""
 	responseId := fmt.Sprintf("chatcmpl-%s", random.GetUUID())
 	createdTime := helper.GetTimestamp()
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
-	go func() {
-		responseBody, err := io.ReadAll(resp.Body)
-		if err != nil {
-			logger.SysError("error reading stream response: " + err.Error())
-			stopChan <- true
-			return
-		}
-		err = resp.Body.Close()
-		if err != nil {
-			logger.SysError("error closing stream response: " + err.Error())
-			stopChan <- true
-			return
-		}
-		var palmResponse ChatResponse
-		err = json.Unmarshal(responseBody, &palmResponse)
-		if err != nil {
-			logger.SysError("error unmarshalling stream response: " + err.Error())
-			stopChan <- true
-			return
-		}
-		fullTextResponse := streamResponsePaLM2OpenAI(&palmResponse)
-		fullTextResponse.Id = responseId
-		fullTextResponse.Created = createdTime
-		if len(palmResponse.Candidates) > 0 {
-			responseText = palmResponse.Candidates[0].Content
-		}
-		jsonResponse, err := json.Marshal(fullTextResponse)
-		if err != nil {
-			logger.SysError("error marshalling stream response: " + err.Error())
-			stopChan <- true
-			return
-		}
-		dataChan <- string(jsonResponse)
-		stopChan <- true
-	}()
+
 	common.SetEventStreamHeaders(c)
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			c.Render(-1, common.CustomEvent{Data: "data: " + data})
-			return true
-		case <-stopChan:
-			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
-			return false
+
+	responseBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		logger.SysError("error reading stream response: " + err.Error())
+		err := resp.Body.Close()
+		if err != nil {
+			return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), ""
 		}
-	})
-	err := resp.Body.Close()
+		return openai.ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError), ""
+	}
+
+	err = resp.Body.Close()
 	if err != nil {
 		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), ""
 	}
+
+	var palmResponse ChatResponse
+	err = json.Unmarshal(responseBody, &palmResponse)
+	if err != nil {
+		logger.SysError("error unmarshalling stream response: " + err.Error())
+		return openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError), ""
+	}
+
+	fullTextResponse := streamResponsePaLM2OpenAI(&palmResponse)
+	fullTextResponse.Id = responseId
+	fullTextResponse.Created = createdTime
+	if len(palmResponse.Candidates) > 0 {
+		responseText = palmResponse.Candidates[0].Content
+	}
+
+	jsonResponse, err := json.Marshal(fullTextResponse)
+	if err != nil {
+		logger.SysError("error marshalling stream response: " + err.Error())
+		return openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError), ""
+	}
+
+	err = render.ObjectData(c, string(jsonResponse))
+	if err != nil {
+		logger.SysError(err.Error())
+	}
+
+	render.Done(c)
+
 	return nil, responseText
 }
 

relay/adaptor/proxy/adaptor.go

@@ -0,0 +1,89 @@
+package proxy
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+	"github.com/pkg/errors"
+	"github.com/songquanpeng/one-api/relay/adaptor"
+	channelhelper "github.com/songquanpeng/one-api/relay/adaptor"
+	"github.com/songquanpeng/one-api/relay/meta"
+	"github.com/songquanpeng/one-api/relay/model"
+	relaymodel "github.com/songquanpeng/one-api/relay/model"
+)
+
+var _ adaptor.Adaptor = new(Adaptor)
+
+const channelName = "proxy"
+
+type Adaptor struct{}
+
+func (a *Adaptor) Init(meta *meta.Meta) {
+}
+
+func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error) {
+	return nil, errors.New("notimplement")
+}
+
+func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
+	for k, v := range resp.Header {
+		for _, vv := range v {
+			c.Writer.Header().Set(k, vv)
+		}
+	}
+
+	c.Writer.WriteHeader(resp.StatusCode)
+	if _, gerr := io.Copy(c.Writer, resp.Body); gerr != nil {
+		return nil, &relaymodel.ErrorWithStatusCode{
+			StatusCode: http.StatusInternalServerError,
+			Error: relaymodel.Error{
+				Message: gerr.Error(),
+			},
+		}
+	}
+
+	return nil, nil
+}
+
+func (a *Adaptor) GetModelList() (models []string) {
+	return nil
+}
+
+func (a *Adaptor) GetChannelName() string {
+	return channelName
+}
+
+// GetRequestURL remove static prefix, and return the real request url to the upstream service
+func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
+	prefix := fmt.Sprintf("/v1/oneapi/proxy/%d", meta.ChannelId)
+	return meta.BaseURL + strings.TrimPrefix(meta.RequestURLPath, prefix), nil
+
+}
+
+func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *meta.Meta) error {
+	for k, v := range c.Request.Header {
+		req.Header.Set(k, v[0])
+	}
+
+	// remove unnecessary headers
+	req.Header.Del("Host")
+	req.Header.Del("Content-Length")
+	req.Header.Del("Accept-Encoding")
+	req.Header.Del("Connection")
+
+	// set authorization header
+	req.Header.Set("Authorization", meta.APIKey)
+
+	return nil
+}
+
+func (a *Adaptor) ConvertImageRequest(request *model.ImageRequest) (any, error) {
+	return nil, errors.Errorf("not implement")
+}
+
+func (a *Adaptor) DoRequest(c *gin.Context, meta *meta.Meta, requestBody io.Reader) (*http.Response, error) {
+	return channelhelper.DoRequestHelper(a, c, meta, requestBody)
+}

relay/adaptor/siliconflow/constants.go

@@ -0,0 +1,36 @@
+package siliconflow
+
+// https://docs.siliconflow.cn/docs/getting-started
+
+var ModelList = []string{
+	"deepseek-ai/deepseek-llm-67b-chat",
+	"Qwen/Qwen1.5-14B-Chat",
+	"Qwen/Qwen1.5-7B-Chat",
+	"Qwen/Qwen1.5-110B-Chat",
+	"Qwen/Qwen1.5-32B-Chat",
+	"01-ai/Yi-1.5-6B-Chat",
+	"01-ai/Yi-1.5-9B-Chat-16K",
+	"01-ai/Yi-1.5-34B-Chat-16K",
+	"THUDM/chatglm3-6b",
+	"deepseek-ai/DeepSeek-V2-Chat",
+	"THUDM/glm-4-9b-chat",
+	"Qwen/Qwen2-72B-Instruct",
+	"Qwen/Qwen2-7B-Instruct",
+	"Qwen/Qwen2-57B-A14B-Instruct",
+	"deepseek-ai/DeepSeek-Coder-V2-Instruct",
+	"Qwen/Qwen2-1.5B-Instruct",
+	"internlm/internlm2_5-7b-chat",
+	"BAAI/bge-large-en-v1.5",
+	"BAAI/bge-large-zh-v1.5",
+	"Pro/Qwen/Qwen2-7B-Instruct",
+	"Pro/Qwen/Qwen2-1.5B-Instruct",
+	"Pro/Qwen/Qwen1.5-7B-Chat",
+	"Pro/THUDM/glm-4-9b-chat",
+	"Pro/THUDM/chatglm3-6b",
+	"Pro/01-ai/Yi-1.5-9B-Chat-16K",
+	"Pro/01-ai/Yi-1.5-6B-Chat",
+	"Pro/google/gemma-2-9b-it",
+	"Pro/internlm/internlm2_5-7b-chat",
+	"Pro/meta-llama/Meta-Llama-3-8B-Instruct",
+	"Pro/mistralai/Mistral-7B-Instruct-v0.2",
+}

relay/adaptor/tencent/adaptor.go

@@ -2,35 +2,43 @@ package tencent
 
 import (
 	"errors"
-	"fmt"
 	"github.com/gin-gonic/gin"
+	"github.com/songquanpeng/one-api/common/helper"
 	"github.com/songquanpeng/one-api/relay/adaptor"
 	"github.com/songquanpeng/one-api/relay/adaptor/openai"
 	"github.com/songquanpeng/one-api/relay/meta"
 	"github.com/songquanpeng/one-api/relay/model"
 	"io"
 	"net/http"
+	"strconv"
 	"strings"
 )
 
 // https://cloud.tencent.com/document/api/1729/101837
 
 type Adaptor struct {
-	Sign string
+	Sign      string
+	Action    string
+	Version   string
+	Timestamp int64
 }
 
 func (a *Adaptor) Init(meta *meta.Meta) {
-
+	a.Action = "ChatCompletions"
+	a.Version = "2023-09-01"
+	a.Timestamp = helper.GetTimestamp()
 }
 
 func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
-	return fmt.Sprintf("%s/hyllm/v1/chat/completions", meta.BaseURL), nil
+	return meta.BaseURL + "/", nil
 }
 
 func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *meta.Meta) error {
 	adaptor.SetupCommonRequestHeader(c, req, meta)
 	req.Header.Set("Authorization", a.Sign)
-	req.Header.Set("X-TC-Action", meta.ActualModelName)
+	req.Header.Set("X-TC-Action", a.Action)
+	req.Header.Set("X-TC-Version", a.Version)
+	req.Header.Set("X-TC-Timestamp", strconv.FormatInt(a.Timestamp, 10))
 	return nil
 }
 
@@ -40,15 +48,13 @@ func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.G
 	}
 	apiKey := c.Request.Header.Get("Authorization")
 	apiKey = strings.TrimPrefix(apiKey, "Bearer ")
-	appId, secretId, secretKey, err := ParseConfig(apiKey)
+	_, secretId, secretKey, err := ParseConfig(apiKey)
 	if err != nil {
 		return nil, err
 	}
 	tencentRequest := ConvertRequest(*request)
-	tencentRequest.AppId = appId
-	tencentRequest.SecretId = secretId
 	// we have to calculate the sign here
-	a.Sign = GetSign(*tencentRequest, secretKey)
+	a.Sign = GetSign(*tencentRequest, a, secretId, secretKey)
 	return tencentRequest, nil
 }
 

relay/adaptor/tencent/constants.go

@@ -1,7 +1,8 @@
 package tencent
 
 var ModelList = []string{
-	"ChatPro",
-	"ChatStd",
-	"hunyuan",
+	"hunyuan-lite",
+	"hunyuan-standard",
+	"hunyuan-standard-256K",
+	"hunyuan-pro",
 }

relay/adaptor/tencent/main.go

@@ -3,11 +3,18 @@ package tencent
 import (
 	"bufio"
 	"crypto/hmac"
-	"crypto/sha1"
-	"encoding/base64"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/songquanpeng/one-api/common/render"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/common"
 	"github.com/songquanpeng/one-api/common/conv"
@@ -17,36 +24,23 @@ import (
 	"github.com/songquanpeng/one-api/relay/adaptor/openai"
 	"github.com/songquanpeng/one-api/relay/constant"
 	"github.com/songquanpeng/one-api/relay/model"
-	"io"
-	"net/http"
-	"sort"
-	"strconv"
-	"strings"
 )
 
-// https://cloud.tencent.com/document/product/1729/97732
-
 func ConvertRequest(request model.GeneralOpenAIRequest) *ChatRequest {
-	messages := make([]Message, 0, len(request.Messages))
+	messages := make([]*Message, 0, len(request.Messages))
 	for i := 0; i < len(request.Messages); i++ {
 		message := request.Messages[i]
-		messages = append(messages, Message{
+		messages = append(messages, &Message{
 			Content: message.StringContent(),
 			Role:    message.Role,
 		})
 	}
-	stream := 0
-	if request.Stream {
-		stream = 1
-	}
 	return &ChatRequest{
-		Timestamp:   helper.GetTimestamp(),
-		Expired:     helper.GetTimestamp() + 24*60*60,
-		QueryID:     random.GetUUID(),
-		Temperature: request.Temperature,
-		TopP:        request.TopP,
-		Stream:      stream,
+		Model:       &request.Model,
+		Stream:      &request.Stream,
 		Messages:    messages,
+		TopP:        &request.TopP,
+		Temperature: &request.Temperature,
 	}
 }
 
@@ -54,7 +48,11 @@ func responseTencent2OpenAI(response *ChatResponse) *openai.TextResponse {
 	fullTextResponse := openai.TextResponse{
 		Object:  "chat.completion",
 		Created: helper.GetTimestamp(),
-		Usage:   response.Usage,
+		Usage: model.Usage{
+			PromptTokens:     response.Usage.PromptTokens,
+			CompletionTokens: response.Usage.CompletionTokens,
+			TotalTokens:      response.Usage.TotalTokens,
+		},
 	}
 	if len(response.Choices) > 0 {
 		choice := openai.TextResponseChoice{
@@ -91,69 +89,52 @@ func streamResponseTencent2OpenAI(TencentResponse *ChatResponse) *openai.ChatCom
 func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusCode, string) {
 	var responseText string
 	scanner := bufio.NewScanner(resp.Body)
-	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
-		if atEOF && len(data) == 0 {
-			return 0, nil, nil
-		}
-		if i := strings.Index(string(data), "\n"); i >= 0 {
-			return i + 1, data[0:i], nil
-		}
-		if atEOF {
-			return len(data), data, nil
-		}
-		return 0, nil, nil
-	})
-	dataChan := make(chan string)
-	stopChan := make(chan bool)
-	go func() {
-		for scanner.Scan() {
-			data := scanner.Text()
-			if len(data) < 5 { // ignore blank line or wrong format
-				continue
-			}
-			if data[:5] != "data:" {
-				continue
-			}
-			data = data[5:]
-			dataChan <- data
-		}
-		stopChan <- true
-	}()
+	scanner.Split(bufio.ScanLines)
+
 	common.SetEventStreamHeaders(c)
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			var TencentResponse ChatResponse
-			err := json.Unmarshal([]byte(data), &TencentResponse)
-			if err != nil {
-				logger.SysError("error unmarshalling stream response: " + err.Error())
-				return true
-			}
-			response := streamResponseTencent2OpenAI(&TencentResponse)
-			if len(response.Choices) != 0 {
-				responseText += conv.AsString(response.Choices[0].Delta.Content)
-			}
-			jsonResponse, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonResponse)})
-			return true
-		case <-stopChan:
-			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
-			return false
+
+	for scanner.Scan() {
+		data := scanner.Text()
+		if len(data) < 5 || !strings.HasPrefix(data, "data:") {
+			continue
 		}
-	})
+		data = strings.TrimPrefix(data, "data:")
+
+		var tencentResponse ChatResponse
+		err := json.Unmarshal([]byte(data), &tencentResponse)
+		if err != nil {
+			logger.SysError("error unmarshalling stream response: " + err.Error())
+			continue
+		}
+
+		response := streamResponseTencent2OpenAI(&tencentResponse)
+		if len(response.Choices) != 0 {
+			responseText += conv.AsString(response.Choices[0].Delta.Content)
+		}
+
+		err = render.ObjectData(c, response)
+		if err != nil {
+			logger.SysError(err.Error())
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	render.Done(c)
+
 	err := resp.Body.Close()
 	if err != nil {
 		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), ""
 	}
+
 	return nil, responseText
 }
 
 func Handler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusCode, *model.Usage) {
 	var TencentResponse ChatResponse
+	var responseP ChatResponseP
 	responseBody, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return openai.ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError), nil
@@ -162,10 +143,11 @@ func Handler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusCode, *
 	if err != nil {
 		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil
 	}
-	err = json.Unmarshal(responseBody, &TencentResponse)
+	err = json.Unmarshal(responseBody, &responseP)
 	if err != nil {
 		return openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError), nil
 	}
+	TencentResponse = responseP.Response
 	if TencentResponse.Error.Code != 0 {
 		return &model.ErrorWithStatusCode{
 			Error: model.Error{
@@ -202,29 +184,62 @@ func ParseConfig(config string) (appId int64, secretId string, secretKey string,
 	return
 }
 
-func GetSign(req ChatRequest, secretKey string) string {
-	params := make([]string, 0)
-	params = append(params, "app_id="+strconv.FormatInt(req.AppId, 10))
-	params = append(params, "secret_id="+req.SecretId)
-	params = append(params, "timestamp="+strconv.FormatInt(req.Timestamp, 10))
-	params = append(params, "query_id="+req.QueryID)
-	params = append(params, "temperature="+strconv.FormatFloat(req.Temperature, 'f', -1, 64))
-	params = append(params, "top_p="+strconv.FormatFloat(req.TopP, 'f', -1, 64))
-	params = append(params, "stream="+strconv.Itoa(req.Stream))
-	params = append(params, "expired="+strconv.FormatInt(req.Expired, 10))
-
-	var messageStr string
-	for _, msg := range req.Messages {
-		messageStr += fmt.Sprintf(`{"role":"%s","content":"%s"},`, msg.Role, msg.Content)
-	}
-	messageStr = strings.TrimSuffix(messageStr, ",")
-	params = append(params, "messages=["+messageStr+"]")
-
-	sort.Strings(params)
-	url := "hunyuan.cloud.tencent.com/hyllm/v1/chat/completions?" + strings.Join(params, "&")
-	mac := hmac.New(sha1.New, []byte(secretKey))
-	signURL := url
-	mac.Write([]byte(signURL))
-	sign := mac.Sum([]byte(nil))
-	return base64.StdEncoding.EncodeToString(sign)
+func sha256hex(s string) string {
+	b := sha256.Sum256([]byte(s))
+	return hex.EncodeToString(b[:])
+}
+
+func hmacSha256(s, key string) string {
+	hashed := hmac.New(sha256.New, []byte(key))
+	hashed.Write([]byte(s))
+	return string(hashed.Sum(nil))
+}
+
+func GetSign(req ChatRequest, adaptor *Adaptor, secId, secKey string) string {
+	// build canonical request string
+	host := "hunyuan.tencentcloudapi.com"
+	httpRequestMethod := "POST"
+	canonicalURI := "/"
+	canonicalQueryString := ""
+	canonicalHeaders := fmt.Sprintf("content-type:%s\nhost:%s\nx-tc-action:%s\n",
+		"application/json", host, strings.ToLower(adaptor.Action))
+	signedHeaders := "content-type;host;x-tc-action"
+	payload, _ := json.Marshal(req)
+	hashedRequestPayload := sha256hex(string(payload))
+	canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s",
+		httpRequestMethod,
+		canonicalURI,
+		canonicalQueryString,
+		canonicalHeaders,
+		signedHeaders,
+		hashedRequestPayload)
+	// build string to sign
+	algorithm := "TC3-HMAC-SHA256"
+	requestTimestamp := strconv.FormatInt(adaptor.Timestamp, 10)
+	timestamp, _ := strconv.ParseInt(requestTimestamp, 10, 64)
+	t := time.Unix(timestamp, 0).UTC()
+	// must be the format 2006-01-02, ref to package time for more info
+	date := t.Format("2006-01-02")
+	credentialScope := fmt.Sprintf("%s/%s/tc3_request", date, "hunyuan")
+	hashedCanonicalRequest := sha256hex(canonicalRequest)
+	string2sign := fmt.Sprintf("%s\n%s\n%s\n%s",
+		algorithm,
+		requestTimestamp,
+		credentialScope,
+		hashedCanonicalRequest)
+
+	// sign string
+	secretDate := hmacSha256(date, "TC3"+secKey)
+	secretService := hmacSha256("hunyuan", secretDate)
+	secretKey := hmacSha256("tc3_request", secretService)
+	signature := hex.EncodeToString([]byte(hmacSha256(string2sign, secretKey)))
+
+	// build authorization
+	authorization := fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s",
+		algorithm,
+		secId,
+		credentialScope,
+		signedHeaders,
+		signature)
+	return authorization
 }

relay/adaptor/tencent/model.go

@@ -1,63 +1,75 @@
 package tencent
 
-import (
-	"github.com/songquanpeng/one-api/relay/model"
-)
-
 type Message struct {
-	Role    string `json:"role"`
-	Content string `json:"content"`
+	Role    string `json:"Role"`
+	Content string `json:"Content"`
 }
 
 type ChatRequest struct {
-	AppId    int64  `json:"app_id"`    // 腾讯云账号的 APPID
-	SecretId string `json:"secret_id"` // 官网 SecretId
-	// Timestamp当前 UNIX 时间戳，单位为秒，可记录发起 API 请求的时间。
-	// 例如1529223702，如果与当前时间相差过大，会引起签名过期错误
-	Timestamp int64 `json:"timestamp"`
-	// Expired 签名的有效期，是一个符合 UNIX Epoch 时间戳规范的数值，
-	// 单位为秒；Expired 必须大于 Timestamp 且 Expired-Timestamp 小于90天
-	Expired int64  `json:"expired"`
-	QueryID string `json:"query_id"` //请求 Id，用于问题排查
-	// Temperature 较高的数值会使输出更加随机，而较低的数值会使其更加集中和确定
-	// 默认 1.0，取值区间为[0.0,2.0]，非必要不建议使用,不合理的取值会影响效果
-	// 建议该参数和 top_p 只设置1个，不要同时更改 top_p
-	Temperature float64 `json:"temperature"`
-	// TopP 影响输出文本的多样性，取值越大，生成文本的多样性越强
-	// 默认1.0，取值区间为[0.0, 1.0]，非必要不建议使用, 不合理的取值会影响效果
-	// 建议该参数和 temperature 只设置1个，不要同时更改
-	TopP float64 `json:"top_p"`
-	// Stream 0：同步，1：流式 （默认，协议：SSE)
-	// 同步请求超时：60s，如果内容较长建议使用流式
-	Stream int `json:"stream"`
-	// Messages 会话内容, 长度最多为40, 按对话时间从旧到新在数组中排列
-	// 输入 content 总数最大支持 3000 token。
-	Messages []Message `json:"messages"`
+	// 模型名称，可选值包括 hunyuan-lite、hunyuan-standard、hunyuan-standard-256K、hunyuan-pro。
+	// 各模型介绍请阅读 [产品概述](https://cloud.tencent.com/document/product/1729/104753) 中的说明。
+	//
+	// 注意：
+	// 不同的模型计费不同，请根据 [购买指南](https://cloud.tencent.com/document/product/1729/97731) 按需调用。
+	Model *string `json:"Model"`
+	// 聊天上下文信息。
+	// 说明：
+	// 1. 长度最多为 40，按对话时间从旧到新在数组中排列。
+	// 2. Message.Role 可选值：system、user、assistant。
+	// 其中，system 角色可选，如存在则必须位于列表的最开始。user 和 assistant 需交替出现（一问一答），以 user 提问开始和结束，且 Content 不能为空。Role 的顺序示例：[system（可选） user assistant user assistant user ...]。
+	// 3. Messages 中 Content 总长度不能超过模型输入长度上限（可参考 [产品概述](https://cloud.tencent.com/document/product/1729/104753) 文档），超过则会截断最前面的内容，只保留尾部内容。
+	Messages []*Message `json:"Messages"`
+	// 流式调用开关。
+	// 说明：
+	// 1. 未传值时默认为非流式调用（false）。
+	// 2. 流式调用时以 SSE 协议增量返回结果（返回值取 Choices[n].Delta 中的值，需要拼接增量数据才能获得完整结果）。
+	// 3. 非流式调用时：
+	// 调用方式与普通 HTTP 请求无异。
+	// 接口响应耗时较长，**如需更低时延建议设置为 true**。
+	// 只返回一次最终结果（返回值取 Choices[n].Message 中的值）。
+	//
+	// 注意：
+	// 通过 SDK 调用时，流式和非流式调用需用**不同的方式**获取返回值，具体参考 SDK 中的注释或示例（在各语言 SDK 代码仓库的 examples/hunyuan/v20230901/ 目录中）。
+	Stream *bool `json:"Stream"`
+	// 说明：
+	// 1. 影响输出文本的多样性，取值越大，生成文本的多样性越强。
+	// 2. 取值区间为 [0.0, 1.0]，未传值时使用各模型推荐值。
+	// 3. 非必要不建议使用，不合理的取值会影响效果。
+	TopP *float64 `json:"TopP"`
+	// 说明：
+	// 1. 较高的数值会使输出更加随机，而较低的数值会使其更加集中和确定。
+	// 2. 取值区间为 [0.0, 2.0]，未传值时使用各模型推荐值。
+	// 3. 非必要不建议使用，不合理的取值会影响效果。
+	Temperature *float64 `json:"Temperature"`
 }
 
 type Error struct {
-	Code    int    `json:"code"`
-	Message string `json:"message"`
+	Code    int    `json:"Code"`
+	Message string `json:"Message"`
 }
 
 type Usage struct {
-	InputTokens  int `json:"input_tokens"`
-	OutputTokens int `json:"output_tokens"`
-	TotalTokens  int `json:"total_tokens"`
+	PromptTokens     int `json:"PromptTokens"`
+	CompletionTokens int `json:"CompletionTokens"`
+	TotalTokens      int `json:"TotalTokens"`
 }
 
 type ResponseChoices struct {
-	FinishReason string  `json:"finish_reason,omitempty"` // 流式结束标志位，为 stop 则表示尾包
-	Messages     Message `json:"messages,omitempty"`      // 内容，同步模式返回内容，流模式为 null 输出 content 内容总数最多支持 1024token。
-	Delta        Message `json:"delta,omitempty"`         // 内容，流模式返回内容，同步模式为 null 输出 content 内容总数最多支持 1024token。
+	FinishReason string  `json:"FinishReason,omitempty"` // 流式结束标志位，为 stop 则表示尾包
+	Messages     Message `json:"Message,omitempty"`      // 内容，同步模式返回内容，流模式为 null 输出 content 内容总数最多支持 1024token。
+	Delta        Message `json:"Delta,omitempty"`        // 内容，流模式返回内容，同步模式为 null 输出 content 内容总数最多支持 1024token。
 }
 
 type ChatResponse struct {
-	Choices []ResponseChoices `json:"choices,omitempty"` // 结果
-	Created string            `json:"created,omitempty"` // unix 时间戳的字符串
-	Id      string            `json:"id,omitempty"`      // 会话 id
-	Usage   model.Usage       `json:"usage,omitempty"`   // token 数量
-	Error   Error             `json:"error,omitempty"`   // 错误信息 注意：此字段可能返回 null，表示取不到有效值
-	Note    string            `json:"note,omitempty"`    // 注释
-	ReqID   string            `json:"req_id,omitempty"`  // 唯一请求 Id，每次请求都会返回。用于反馈接口入参
+	Choices []ResponseChoices `json:"Choices,omitempty"` // 结果
+	Created int64             `json:"Created,omitempty"` // unix 时间戳的字符串
+	Id      string            `json:"Id,omitempty"`      // 会话 id
+	Usage   Usage             `json:"Usage,omitempty"`   // token 数量
+	Error   Error             `json:"Error,omitempty"`   // 错误信息 注意：此字段可能返回 null，表示取不到有效值
+	Note    string            `json:"Note,omitempty"`    // 注释
+	ReqID   string            `json:"Req_id,omitempty"`  // 唯一请求 Id，每次请求都会返回。用于反馈接口入参
+}
+
+type ChatResponseP struct {
+	Response ChatResponse `json:"Response,omitempty"`
 }

relay/adaptor/vertexai/adaptor.go

@@ -0,0 +1,117 @@
+package vertexai
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+	"github.com/songquanpeng/one-api/relay/adaptor"
+	channelhelper "github.com/songquanpeng/one-api/relay/adaptor"
+	"github.com/songquanpeng/one-api/relay/meta"
+	"github.com/songquanpeng/one-api/relay/model"
+	relaymodel "github.com/songquanpeng/one-api/relay/model"
+)
+
+var _ adaptor.Adaptor = new(Adaptor)
+
+const channelName = "vertexai"
+
+type Adaptor struct{}
+
+func (a *Adaptor) Init(meta *meta.Meta) {
+}
+
+func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error) {
+	if request == nil {
+		return nil, errors.New("request is nil")
+	}
+
+	adaptor := GetAdaptor(request.Model)
+	if adaptor == nil {
+		return nil, errors.New("adaptor not found")
+	}
+
+	return adaptor.ConvertRequest(c, relayMode, request)
+}
+
+func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
+	adaptor := GetAdaptor(meta.ActualModelName)
+	if adaptor == nil {
+		return nil, &relaymodel.ErrorWithStatusCode{
+			StatusCode: http.StatusInternalServerError,
+			Error: relaymodel.Error{
+				Message: "adaptor not found",
+			},
+		}
+	}
+	return adaptor.DoResponse(c, resp, meta)
+}
+
+func (a *Adaptor) GetModelList() (models []string) {
+	models = modelList
+	return
+}
+
+func (a *Adaptor) GetChannelName() string {
+	return channelName
+}
+
+func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
+	suffix := ""
+	if strings.HasPrefix(meta.ActualModelName, "gemini") {
+		if meta.IsStream {
+			suffix = "streamGenerateContent?alt=sse"
+		} else {
+			suffix = "generateContent"
+		}
+	} else {
+		if meta.IsStream {
+			suffix = "streamRawPredict?alt=sse"
+		} else {
+			suffix = "rawPredict"
+		}
+	}
+
+	if meta.BaseURL != "" {
+		return fmt.Sprintf(
+			"%s/v1/projects/%s/locations/%s/publishers/google/models/%s:%s",
+			meta.BaseURL,
+			meta.Config.VertexAIProjectID,
+			meta.Config.Region,
+			meta.ActualModelName,
+			suffix,
+		), nil
+	}
+	return fmt.Sprintf(
+		"https://%s-aiplatform.googleapis.com/v1/projects/%s/locations/%s/publishers/google/models/%s:%s",
+		meta.Config.Region,
+		meta.Config.VertexAIProjectID,
+		meta.Config.Region,
+		meta.ActualModelName,
+		suffix,
+	), nil
+}
+
+func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *meta.Meta) error {
+	adaptor.SetupCommonRequestHeader(c, req, meta)
+	token, err := getToken(c, meta.ChannelId, meta.Config.VertexAIADC)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	return nil
+}
+
+func (a *Adaptor) ConvertImageRequest(request *model.ImageRequest) (any, error) {
+	if request == nil {
+		return nil, errors.New("request is nil")
+	}
+	return request, nil
+}
+
+func (a *Adaptor) DoRequest(c *gin.Context, meta *meta.Meta, requestBody io.Reader) (*http.Response, error) {
+	return channelhelper.DoRequestHelper(a, c, meta, requestBody)
+}

relay/adaptor/vertexai/claude/adapter.go

@@ -0,0 +1,55 @@
+package vertexai
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/pkg/errors"
+	"github.com/songquanpeng/one-api/common/ctxkey"
+	"github.com/songquanpeng/one-api/relay/adaptor/anthropic"
+
+	"github.com/songquanpeng/one-api/relay/meta"
+	"github.com/songquanpeng/one-api/relay/model"
+)
+
+var ModelList = []string{
+	"claude-3-haiku@20240307", "claude-3-opus@20240229", "claude-3-5-sonnet@20240620", "claude-3-sonnet@20240229",
+}
+
+const anthropicVersion = "vertex-2023-10-16"
+
+type Adaptor struct {
+}
+
+func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error) {
+	if request == nil {
+		return nil, errors.New("request is nil")
+	}
+
+	claudeReq := anthropic.ConvertRequest(*request)
+	req := Request{
+		AnthropicVersion: anthropicVersion,
+		// Model:            claudeReq.Model,
+		Messages:    claudeReq.Messages,
+		System:      claudeReq.System,
+		MaxTokens:   claudeReq.MaxTokens,
+		Temperature: claudeReq.Temperature,
+		TopP:        claudeReq.TopP,
+		TopK:        claudeReq.TopK,
+		Stream:      claudeReq.Stream,
+		Tools:       claudeReq.Tools,
+	}
+
+	c.Set(ctxkey.RequestModel, request.Model)
+	c.Set(ctxkey.ConvertedRequest, req)
+	return req, nil
+}
+
+func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
+	if meta.IsStream {
+		err, usage = anthropic.StreamHandler(c, resp)
+	} else {
+		err, usage = anthropic.Handler(c, resp, meta.PromptTokens, meta.ActualModelName)
+	}
+	return
+}

relay/adaptor/vertexai/claude/model.go

@@ -0,0 +1,19 @@
+package vertexai
+
+import "github.com/songquanpeng/one-api/relay/adaptor/anthropic"
+
+type Request struct {
+	// AnthropicVersion must be "vertex-2023-10-16"
+	AnthropicVersion string `json:"anthropic_version"`
+	// Model            string              `json:"model"`
+	Messages      []anthropic.Message `json:"messages"`
+	System        string              `json:"system,omitempty"`
+	MaxTokens     int                 `json:"max_tokens,omitempty"`
+	StopSequences []string            `json:"stop_sequences,omitempty"`
+	Stream        bool                `json:"stream,omitempty"`
+	Temperature   float64             `json:"temperature,omitempty"`
+	TopP          float64             `json:"top_p,omitempty"`
+	TopK          int                 `json:"top_k,omitempty"`
+	Tools         []anthropic.Tool    `json:"tools,omitempty"`
+	ToolChoice    any                 `json:"tool_choice,omitempty"`
+}

relay/adaptor/vertexai/gemini/adapter.go

@@ -0,0 +1,49 @@
+package vertexai
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/pkg/errors"
+	"github.com/songquanpeng/one-api/common/ctxkey"
+	"github.com/songquanpeng/one-api/relay/adaptor/gemini"
+	"github.com/songquanpeng/one-api/relay/adaptor/openai"
+	"github.com/songquanpeng/one-api/relay/relaymode"
+
+	"github.com/songquanpeng/one-api/relay/meta"
+	"github.com/songquanpeng/one-api/relay/model"
+)
+
+var ModelList = []string{
+	"gemini-1.5-pro-001", "gemini-1.5-flash-001", "gemini-pro", "gemini-pro-vision",
+}
+
+type Adaptor struct {
+}
+
+func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error) {
+	if request == nil {
+		return nil, errors.New("request is nil")
+	}
+
+	geminiRequest := gemini.ConvertRequest(*request)
+	c.Set(ctxkey.RequestModel, request.Model)
+	c.Set(ctxkey.ConvertedRequest, geminiRequest)
+	return geminiRequest, nil
+}
+
+func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
+	if meta.IsStream {
+		var responseText string
+		err, responseText = gemini.StreamHandler(c, resp)
+		usage = openai.ResponseText2Usage(responseText, meta.ActualModelName, meta.PromptTokens)
+	} else {
+		switch meta.Mode {
+		case relaymode.Embeddings:
+			err, usage = gemini.EmbeddingHandler(c, resp)
+		default:
+			err, usage = gemini.Handler(c, resp, meta.PromptTokens, meta.ActualModelName)
+		}
+	}
+	return
+}

relay/adaptor/vertexai/registry.go

@@ -0,0 +1,50 @@
+package vertexai
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	claude "github.com/songquanpeng/one-api/relay/adaptor/vertexai/claude"
+	gemini "github.com/songquanpeng/one-api/relay/adaptor/vertexai/gemini"
+	"github.com/songquanpeng/one-api/relay/meta"
+	"github.com/songquanpeng/one-api/relay/model"
+)
+
+type VertexAIModelType int
+
+const (
+	VerterAIClaude VertexAIModelType = iota + 1
+	VerterAIGemini
+)
+
+var modelMapping = map[string]VertexAIModelType{}
+var modelList = []string{}
+
+func init() {
+	modelList = append(modelList, claude.ModelList...)
+	for _, model := range claude.ModelList {
+		modelMapping[model] = VerterAIClaude
+	}
+
+	modelList = append(modelList, gemini.ModelList...)
+	for _, model := range gemini.ModelList {
+		modelMapping[model] = VerterAIGemini
+	}
+}
+
+type innerAIAdapter interface {
+	ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error)
+	DoResponse(c *gin.Context, resp *http.Response, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode)
+}
+
+func GetAdaptor(model string) innerAIAdapter {
+	adaptorType := modelMapping[model]
+	switch adaptorType {
+	case VerterAIClaude:
+		return &claude.Adaptor{}
+	case VerterAIGemini:
+		return &gemini.Adaptor{}
+	default:
+		return nil
+	}
+}

relay/adaptor/vertexai/token.go

@@ -0,0 +1,62 @@
+package vertexai
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	credentials "cloud.google.com/go/iam/credentials/apiv1"
+	"cloud.google.com/go/iam/credentials/apiv1/credentialspb"
+	"github.com/patrickmn/go-cache"
+	"google.golang.org/api/option"
+)
+
+type ApplicationDefaultCredentials struct {
+	Type                    string `json:"type"`
+	ProjectID               string `json:"project_id"`
+	PrivateKeyID            string `json:"private_key_id"`
+	PrivateKey              string `json:"private_key"`
+	ClientEmail             string `json:"client_email"`
+	ClientID                string `json:"client_id"`
+	AuthURI                 string `json:"auth_uri"`
+	TokenURI                string `json:"token_uri"`
+	AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"`
+	ClientX509CertURL       string `json:"client_x509_cert_url"`
+	UniverseDomain          string `json:"universe_domain"`
+}
+
+var Cache = cache.New(50*time.Minute, 55*time.Minute)
+
+const defaultScope = "https://www.googleapis.com/auth/cloud-platform"
+
+func getToken(ctx context.Context, channelId int, adcJson string) (string, error) {
+	cacheKey := fmt.Sprintf("vertexai-token-%d", channelId)
+	if token, found := Cache.Get(cacheKey); found {
+		return token.(string), nil
+	}
+	adc := &ApplicationDefaultCredentials{}
+	if err := json.Unmarshal([]byte(adcJson), adc); err != nil {
+		return "", fmt.Errorf("Failed to decode credentials file: %w", err)
+	}
+
+	c, err := credentials.NewIamCredentialsClient(ctx, option.WithCredentialsJSON([]byte(adcJson)))
+	if err != nil {
+		return "", fmt.Errorf("Failed to create client: %w", err)
+	}
+	defer c.Close()
+
+	req := &credentialspb.GenerateAccessTokenRequest{
+		// See https://pkg.go.dev/cloud.google.com/go/iam/credentials/apiv1/credentialspb#GenerateAccessTokenRequest.
+		Name:  fmt.Sprintf("projects/-/serviceAccounts/%s", adc.ClientEmail),
+		Scope: []string{defaultScope},
+	}
+	resp, err := c.GenerateAccessToken(ctx, req)
+	if err != nil {
+		return "", fmt.Errorf("Failed to generate access token: %w", err)
+	}
+	_ = resp
+
+	Cache.Set(cacheKey, resp.AccessToken, cache.DefaultExpiration)
+	return resp.AccessToken, nil
+}

relay/adaptor/xunfei/adaptor.go

@@ -27,14 +27,6 @@ func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
 
 func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *meta.Meta) error {
 	adaptor.SetupCommonRequestHeader(c, req, meta)
-	version := parseAPIVersionByModelName(meta.ActualModelName)
-	if version == "" {
-		version = a.meta.Config.APIVersion
-	}
-	if version == "" {
-		version = "v1.1"
-	}
-	a.meta.Config.APIVersion = version
 	// check DoResponse for auth part
 	return nil
 }
@@ -69,6 +61,14 @@ func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *meta.Met
 	if a.request == nil {
 		return nil, openai.ErrorWrapper(errors.New("request is nil"), "request_is_nil", http.StatusBadRequest)
 	}
+	version := parseAPIVersionByModelName(meta.ActualModelName)
+	if version == "" {
+		version = a.meta.Config.APIVersion
+	}
+	if version == "" {
+		version = "v1.1"
+	}
+	a.meta.Config.APIVersion = version
 	if meta.IsStream {
 		err, usage = StreamHandler(c, meta, *a.request, splits[0], splits[1], splits[2])
 	} else {

relay/adaptor/xunfei/constants.go

@@ -6,4 +6,5 @@ var ModelList = []string{
 	"SparkDesk-v2.1",
 	"SparkDesk-v3.1",
 	"SparkDesk-v3.5",
+	"SparkDesk-v4.0",
 }

relay/adaptor/xunfei/main.go

@@ -5,7 +5,14 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"github.com/songquanpeng/one-api/common"
@@ -16,11 +23,6 @@ import (
 	"github.com/songquanpeng/one-api/relay/constant"
 	"github.com/songquanpeng/one-api/relay/meta"
 	"github.com/songquanpeng/one-api/relay/model"
-	"io"
-	"net/http"
-	"net/url"
-	"strings"
-	"time"
 )
 
 // https://console.xfyun.cn/services/cbm
@@ -28,11 +30,7 @@ import (
 
 func requestOpenAI2Xunfei(request model.GeneralOpenAIRequest, xunfeiAppId string, domain string) *ChatRequest {
 	messages := make([]Message, 0, len(request.Messages))
-	var lastToolCalls []model.Tool
 	for _, message := range request.Messages {
-		if message.ToolCalls != nil {
-			lastToolCalls = message.ToolCalls
-		}
 		messages = append(messages, Message{
 			Role:    message.Role,
 			Content: message.StringContent(),
@@ -45,9 +43,14 @@ func requestOpenAI2Xunfei(request model.GeneralOpenAIRequest, xunfeiAppId string
 	xunfeiRequest.Parameter.Chat.TopK = request.N
 	xunfeiRequest.Parameter.Chat.MaxTokens = request.MaxTokens
 	xunfeiRequest.Payload.Message.Text = messages
-	if len(lastToolCalls) != 0 {
-		for _, toolCall := range lastToolCalls {
-			xunfeiRequest.Payload.Functions.Text = append(xunfeiRequest.Payload.Functions.Text, toolCall.Function)
+
+	if strings.HasPrefix(domain, "generalv3") || domain == "4.0Ultra" {
+		functions := make([]model.Function, len(request.Tools))
+		for i, tool := range request.Tools {
+			functions[i] = tool.Function
+		}
+		xunfeiRequest.Payload.Functions = &Functions{
+			Text: functions,
 		}
 	}
 
@@ -203,7 +206,7 @@ func Handler(c *gin.Context, meta *meta.Meta, textRequest model.GeneralOpenAIReq
 		}
 	}
 	if len(xunfeiResponse.Payload.Choices.Text) == 0 {
-		return openai.ErrorWrapper(err, "xunfei_empty_response_detected", http.StatusInternalServerError), nil
+		return openai.ErrorWrapper(errors.New("xunfei empty response detected"), "xunfei_empty_response_detected", http.StatusInternalServerError), nil
 	}
 	xunfeiResponse.Payload.Choices.Text[0].Content = content
 
@@ -287,6 +290,8 @@ func apiVersion2domain(apiVersion string) string {
 		return "generalv3"
 	case "v3.5":
 		return "generalv3.5"
+	case "v4.0":
+		return "4.0Ultra"
 	}
 	return "general" + apiVersion
 }

relay/adaptor/xunfei/model.go

@@ -9,6 +9,10 @@ type Message struct {
 	Content string `json:"content"`
 }
 
+type Functions struct {
+	Text []model.Function `json:"text,omitempty"`
+}
+
 type ChatRequest struct {
 	Header struct {
 		AppId string `json:"app_id"`
@@ -26,9 +30,7 @@ type ChatRequest struct {
 		Message struct {
 			Text []Message `json:"text"`
 		} `json:"message"`
-		Functions struct {
-			Text []model.Function `json:"text,omitempty"`
-		} `json:"functions,omitempty"`
+		Functions *Functions `json:"functions,omitempty"`
 	} `json:"payload"`
 }
 

relay/adaptor/zhipu/main.go

@@ -3,6 +3,13 @@ package zhipu
 import (
 	"bufio"
 	"encoding/json"
+	"github.com/songquanpeng/one-api/common/render"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
 	"github.com/gin-gonic/gin"
 	"github.com/golang-jwt/jwt"
 	"github.com/songquanpeng/one-api/common"
@@ -11,11 +18,6 @@ import (
 	"github.com/songquanpeng/one-api/relay/adaptor/openai"
 	"github.com/songquanpeng/one-api/relay/constant"
 	"github.com/songquanpeng/one-api/relay/model"
-	"io"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
 )
 
 // https://open.bigmodel.cn/doc/api#chatglm_std
@@ -155,66 +157,55 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 		}
 		return 0, nil, nil
 	})
-	dataChan := make(chan string)
-	metaChan := make(chan string)
-	stopChan := make(chan bool)
-	go func() {
-		for scanner.Scan() {
-			data := scanner.Text()
-			lines := strings.Split(data, "\n")
-			for i, line := range lines {
-				if len(line) < 5 {
+
+	common.SetEventStreamHeaders(c)
+
+	for scanner.Scan() {
+		data := scanner.Text()
+		lines := strings.Split(data, "\n")
+		for i, line := range lines {
+			if len(line) < 5 {
+				continue
+			}
+			if strings.HasPrefix(line, "data:") {
+				dataSegment := line[5:]
+				if i != len(lines)-1 {
+					dataSegment += "\n"
+				}
+				response := streamResponseZhipu2OpenAI(dataSegment)
+				err := render.ObjectData(c, response)
+				if err != nil {
+					logger.SysError("error marshalling stream response: " + err.Error())
+				}
+			} else if strings.HasPrefix(line, "meta:") {
+				metaSegment := line[5:]
+				var zhipuResponse StreamMetaResponse
+				err := json.Unmarshal([]byte(metaSegment), &zhipuResponse)
+				if err != nil {
+					logger.SysError("error unmarshalling stream response: " + err.Error())
 					continue
 				}
-				if line[:5] == "data:" {
-					dataChan <- line[5:]
-					if i != len(lines)-1 {
-						dataChan <- "\n"
-					}
-				} else if line[:5] == "meta:" {
-					metaChan <- line[5:]
+				response, zhipuUsage := streamMetaResponseZhipu2OpenAI(&zhipuResponse)
+				err = render.ObjectData(c, response)
+				if err != nil {
+					logger.SysError("error marshalling stream response: " + err.Error())
 				}
+				usage = zhipuUsage
 			}
 		}
-		stopChan <- true
-	}()
-	common.SetEventStreamHeaders(c)
-	c.Stream(func(w io.Writer) bool {
-		select {
-		case data := <-dataChan:
-			response := streamResponseZhipu2OpenAI(data)
-			jsonResponse, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonResponse)})
-			return true
-		case data := <-metaChan:
-			var zhipuResponse StreamMetaResponse
-			err := json.Unmarshal([]byte(data), &zhipuResponse)
-			if err != nil {
-				logger.SysError("error unmarshalling stream response: " + err.Error())
-				return true
-			}
-			response, zhipuUsage := streamMetaResponseZhipu2OpenAI(&zhipuResponse)
-			jsonResponse, err := json.Marshal(response)
-			if err != nil {
-				logger.SysError("error marshalling stream response: " + err.Error())
-				return true
-			}
-			usage = zhipuUsage
-			c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonResponse)})
-			return true
-		case <-stopChan:
-			c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
-			return false
-		}
-	})
+	}
+
+	if err := scanner.Err(); err != nil {
+		logger.SysError("error reading stream: " + err.Error())
+	}
+
+	render.Done(c)
+
 	err := resp.Body.Close()
 	if err != nil {
 		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), nil
 	}
+
 	return nil, usage
 }