Merge remote-tracking branch 'origin/main'

relay/adaptor/anthropic/adaptor.go

@@ -3,12 +3,14 @@ package anthropic
 import (
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/relay/adaptor"
 	"github.com/songquanpeng/one-api/relay/meta"
 	"github.com/songquanpeng/one-api/relay/model"
-	"io"
-	"net/http"
 )
 
 type Adaptor struct {
@@ -31,6 +33,13 @@ func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *me
 	}
 	req.Header.Set("anthropic-version", anthropicVersion)
 	req.Header.Set("anthropic-beta", "messages-2023-12-15")
+
+	// https://x.com/alexalbert__/status/1812921642143900036
+	// claude-3-5-sonnet can support 8k context
+	if strings.HasPrefix(meta.ActualModelName, "claude-3-5-sonnet") {
+		req.Header.Set("anthropic-beta", "max-tokens-3-5-sonnet-2024-07-15")
+	}
+
 	return nil
 }
 

relay/adaptor/cloudflare/constant.go

@@ -1,6 +1,7 @@
 package cloudflare
 
 var ModelList = []string{
+	"@cf/meta/llama-3.1-8b-instruct",
 	"@cf/meta/llama-2-7b-chat-fp16",
 	"@cf/meta/llama-2-7b-chat-int8",
 	"@cf/mistral/mistral-7b-instruct-v0.1",

relay/adaptor/doubao/main.go

@@ -7,8 +7,12 @@ import (
 )
 
 func GetRequestURL(meta *meta.Meta) (string, error) {
-	if meta.Mode == relaymode.ChatCompletions {
+	switch meta.Mode {
+	case relaymode.ChatCompletions:
 		return fmt.Sprintf("%s/api/v3/chat/completions", meta.BaseURL), nil
+	case relaymode.Embeddings:
+		return fmt.Sprintf("%s/api/v3/embeddings", meta.BaseURL), nil
+	default:
 	}
 	return "", fmt.Errorf("unsupported relay mode %d for doubao", meta.Mode)
 }

relay/adaptor/gemini/constants.go

@@ -3,6 +3,5 @@ package gemini
 // https://ai.google.dev/models/gemini
 
 var ModelList = []string{
-	"gemini-pro", "gemini-1.0-pro-001", "gemini-1.5-pro",
-	"gemini-pro-vision", "gemini-1.0-pro-vision-001", "embedding-001", "text-embedding-004",
+	"gemini-pro", "gemini-1.0-pro", "gemini-1.5-flash", "gemini-1.5-pro", "text-embedding-004", "aqa",
 }

relay/adaptor/groq/constants.go

@@ -4,9 +4,14 @@ package groq
 
 var ModelList = []string{
 	"gemma-7b-it",
-	"llama2-7b-2048",
-	"llama2-70b-4096",
 	"mixtral-8x7b-32768",
 	"llama3-8b-8192",
 	"llama3-70b-8192",
+	"gemma2-9b-it",
+	"llama-3.1-405b-reasoning",
+	"llama-3.1-70b-versatile",
+	"llama-3.1-8b-instant",
+	"llama3-groq-70b-8192-tool-use-preview",
+	"llama3-groq-8b-8192-tool-use-preview",
+	"whisper-large-v3",
 }

relay/adaptor/ollama/adaptor.go

@@ -24,7 +24,7 @@ func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
 	// https://github.com/ollama/ollama/blob/main/docs/api.md
 	fullRequestURL := fmt.Sprintf("%s/api/chat", meta.BaseURL)
 	if meta.Mode == relaymode.Embeddings {
-		fullRequestURL = fmt.Sprintf("%s/api/embeddings", meta.BaseURL)
+		fullRequestURL = fmt.Sprintf("%s/api/embed", meta.BaseURL)
 	}
 	return fullRequestURL, nil
 }

relay/adaptor/ollama/main.go

@@ -31,6 +31,8 @@ func ConvertRequest(request model.GeneralOpenAIRequest) *ChatRequest {
 			TopP:             request.TopP,
 			FrequencyPenalty: request.FrequencyPenalty,
 			PresencePenalty:  request.PresencePenalty,
+			NumPredict:  	  request.MaxTokens,
+			NumCtx:  	  request.NumCtx,
 		},
 		Stream: request.Stream,
 	}
@@ -118,8 +120,10 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 	common.SetEventStreamHeaders(c)
 
 	for scanner.Scan() {
-		data := strings.TrimPrefix(scanner.Text(), "}")
-		data = data + "}"
+		data := scanner.Text()
+		if strings.HasPrefix(data, "}") {
+		    data = strings.TrimPrefix(data, "}") + "}"
+		}
 
 		var ollamaResponse ChatResponse
 		err := json.Unmarshal([]byte(data), &ollamaResponse)
@@ -157,8 +161,15 @@ func StreamHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusC
 
 func ConvertEmbeddingRequest(request model.GeneralOpenAIRequest) *EmbeddingRequest {
 	return &EmbeddingRequest{
-		Model:  request.Model,
-		Prompt: strings.Join(request.ParseInput(), " "),
+		Model: request.Model,
+		Input: request.ParseInput(),
+		Options: &Options{
+			Seed:             int(request.Seed),
+			Temperature:      request.Temperature,
+			TopP:             request.TopP,
+			FrequencyPenalty: request.FrequencyPenalty,
+			PresencePenalty:  request.PresencePenalty,
+		},
 	}
 }
 
@@ -201,15 +212,17 @@ func embeddingResponseOllama2OpenAI(response *EmbeddingResponse) *openai.Embeddi
 	openAIEmbeddingResponse := openai.EmbeddingResponse{
 		Object: "list",
 		Data:   make([]openai.EmbeddingResponseItem, 0, 1),
-		Model:  "text-embedding-v1",
+		Model:  response.Model,
 		Usage:  model.Usage{TotalTokens: 0},
 	}
 
-	openAIEmbeddingResponse.Data = append(openAIEmbeddingResponse.Data, openai.EmbeddingResponseItem{
-		Object:    `embedding`,
-		Index:     0,
-		Embedding: response.Embedding,
-	})
+	for i, embedding := range response.Embeddings {
+		openAIEmbeddingResponse.Data = append(openAIEmbeddingResponse.Data, openai.EmbeddingResponseItem{
+			Object:    `embedding`,
+			Index:     i,
+			Embedding: embedding,
+		})
+	}
 	return &openAIEmbeddingResponse
 }
 

relay/adaptor/ollama/model.go

@@ -7,6 +7,8 @@ type Options struct {
 	TopP             float64 `json:"top_p,omitempty"`
 	FrequencyPenalty float64 `json:"frequency_penalty,omitempty"`
 	PresencePenalty  float64 `json:"presence_penalty,omitempty"`
+	NumPredict  	 int 	 `json:"num_predict,omitempty"`
+	NumCtx  	 int 	 `json:"num_ctx,omitempty"`
 }
 
 type Message struct {
@@ -37,11 +39,15 @@ type ChatResponse struct {
 }
 
 type EmbeddingRequest struct {
-	Model  string `json:"model"`
-	Prompt string `json:"prompt"`
+	Model string   `json:"model"`
+	Input []string `json:"input"`
+	// Truncate  bool     `json:"truncate,omitempty"`
+	Options *Options `json:"options,omitempty"`
+	// KeepAlive string   `json:"keep_alive,omitempty"`
 }
 
 type EmbeddingResponse struct {
-	Error     string    `json:"error,omitempty"`
-	Embedding []float64 `json:"embedding,omitempty"`
+	Error      string      `json:"error,omitempty"`
+	Model      string      `json:"model"`
+	Embeddings [][]float64 `json:"embeddings"`
 }

relay/adaptor/openai/compatible.go

@@ -13,6 +13,7 @@ import (
 	"github.com/songquanpeng/one-api/relay/adaptor/novita"
 	"github.com/songquanpeng/one-api/relay/adaptor/stepfun"
 	"github.com/songquanpeng/one-api/relay/adaptor/togetherai"
+	"github.com/songquanpeng/one-api/relay/adaptor/siliconflow"
 	"github.com/songquanpeng/one-api/relay/channeltype"
 )
 
@@ -30,6 +31,7 @@ var CompatibleChannels = []int{
 	channeltype.DeepSeek,
 	channeltype.TogetherAI,
 	channeltype.Novita,
+	channeltype.SiliconFlow,
 }
 
 func GetCompatibleChannelMeta(channelType int) (string, []string) {
@@ -60,6 +62,8 @@ func GetCompatibleChannelMeta(channelType int) (string, []string) {
 		return "doubao", doubao.ModelList
 	case channeltype.Novita:
 		return "novita", novita.ModelList
+	case channeltype.SiliconFlow:
+		return "siliconflow", siliconflow.ModelList
 	default:
 		return "openai", ModelList
 	}

relay/adaptor/openai/constants.go

@@ -8,6 +8,7 @@ var ModelList = []string{
 	"gpt-4-32k", "gpt-4-32k-0314", "gpt-4-32k-0613",
 	"gpt-4-turbo-preview", "gpt-4-turbo", "gpt-4-turbo-2024-04-09",
 	"gpt-4o", "gpt-4o-2024-05-13",
+	"gpt-4o-mini", "gpt-4o-mini-2024-07-18",
 	"gpt-4-vision-preview",
 	"text-embedding-ada-002", "text-embedding-3-small", "text-embedding-3-large",
 	"text-curie-001", "text-babbage-001", "text-ada-001", "text-davinci-002", "text-davinci-003",

relay/adaptor/openai/token.go

@@ -110,7 +110,7 @@ func CountTokenMessages(messages []model.Message, model string) int {
 						if imageUrl["detail"] != nil {
 							detail = imageUrl["detail"].(string)
 						}
-						imageTokens, err := countImageTokens(url, detail)
+						imageTokens, err := countImageTokens(url, detail, model)
 						if err != nil {
 							logger.SysError("error counting image tokens: " + err.Error())
 						} else {
@@ -134,11 +134,15 @@ const (
 	lowDetailCost         = 85
 	highDetailCostPerTile = 170
 	additionalCost        = 85
+	// gpt-4o-mini cost higher than other model
+	gpt4oMiniLowDetailCost  = 2833
+	gpt4oMiniHighDetailCost = 5667
+	gpt4oMiniAdditionalCost = 2833
 )
 
 // https://platform.openai.com/docs/guides/vision/calculating-costs
 // https://github.com/openai/openai-cookbook/blob/05e3f9be4c7a2ae7ecf029a7c32065b024730ebe/examples/How_to_count_tokens_with_tiktoken.ipynb
-func countImageTokens(url string, detail string) (_ int, err error) {
+func countImageTokens(url string, detail string, model string) (_ int, err error) {
 	var fetchSize = true
 	var width, height int
 	// Reference: https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding
@@ -172,6 +176,9 @@ func countImageTokens(url string, detail string) (_ int, err error) {
 	}
 	switch detail {
 	case "low":
+		if strings.HasPrefix(model, "gpt-4o-mini") {
+			return gpt4oMiniLowDetailCost, nil
+		}
 		return lowDetailCost, nil
 	case "high":
 		if fetchSize {
@@ -191,6 +198,9 @@ func countImageTokens(url string, detail string) (_ int, err error) {
 			height = int(float64(height) * ratio)
 		}
 		numSquares := int(math.Ceil(float64(width)/512) * math.Ceil(float64(height)/512))
+		if strings.HasPrefix(model, "gpt-4o-mini") {
+			return numSquares*gpt4oMiniHighDetailCost + gpt4oMiniAdditionalCost, nil
+		}
 		result := numSquares*highDetailCostPerTile + additionalCost
 		return result, nil
 	default:

relay/adaptor/proxy/adaptor.go

@@ -0,0 +1,89 @@
+package proxy
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+	"github.com/pkg/errors"
+	"github.com/songquanpeng/one-api/relay/adaptor"
+	channelhelper "github.com/songquanpeng/one-api/relay/adaptor"
+	"github.com/songquanpeng/one-api/relay/meta"
+	"github.com/songquanpeng/one-api/relay/model"
+	relaymodel "github.com/songquanpeng/one-api/relay/model"
+)
+
+var _ adaptor.Adaptor = new(Adaptor)
+
+const channelName = "proxy"
+
+type Adaptor struct{}
+
+func (a *Adaptor) Init(meta *meta.Meta) {
+}
+
+func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error) {
+	return nil, errors.New("notimplement")
+}
+
+func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
+	for k, v := range resp.Header {
+		for _, vv := range v {
+			c.Writer.Header().Set(k, vv)
+		}
+	}
+
+	c.Writer.WriteHeader(resp.StatusCode)
+	if _, gerr := io.Copy(c.Writer, resp.Body); gerr != nil {
+		return nil, &relaymodel.ErrorWithStatusCode{
+			StatusCode: http.StatusInternalServerError,
+			Error: relaymodel.Error{
+				Message: gerr.Error(),
+			},
+		}
+	}
+
+	return nil, nil
+}
+
+func (a *Adaptor) GetModelList() (models []string) {
+	return nil
+}
+
+func (a *Adaptor) GetChannelName() string {
+	return channelName
+}
+
+// GetRequestURL remove static prefix, and return the real request url to the upstream service
+func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) {
+	prefix := fmt.Sprintf("/v1/oneapi/proxy/%d", meta.ChannelId)
+	return meta.BaseURL + strings.TrimPrefix(meta.RequestURLPath, prefix), nil
+
+}
+
+func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *meta.Meta) error {
+	for k, v := range c.Request.Header {
+		req.Header.Set(k, v[0])
+	}
+
+	// remove unnecessary headers
+	req.Header.Del("Host")
+	req.Header.Del("Content-Length")
+	req.Header.Del("Accept-Encoding")
+	req.Header.Del("Connection")
+
+	// set authorization header
+	req.Header.Set("Authorization", meta.APIKey)
+
+	return nil
+}
+
+func (a *Adaptor) ConvertImageRequest(request *model.ImageRequest) (any, error) {
+	return nil, errors.Errorf("not implement")
+}
+
+func (a *Adaptor) DoRequest(c *gin.Context, meta *meta.Meta, requestBody io.Reader) (*http.Response, error) {
+	return channelhelper.DoRequestHelper(a, c, meta, requestBody)
+}

relay/adaptor/siliconflow/constants.go

@@ -0,0 +1,36 @@
+package siliconflow
+
+// https://docs.siliconflow.cn/docs/getting-started
+
+var ModelList = []string{
+	"deepseek-ai/deepseek-llm-67b-chat",
+	"Qwen/Qwen1.5-14B-Chat",
+	"Qwen/Qwen1.5-7B-Chat",
+	"Qwen/Qwen1.5-110B-Chat",
+	"Qwen/Qwen1.5-32B-Chat",
+	"01-ai/Yi-1.5-6B-Chat",
+	"01-ai/Yi-1.5-9B-Chat-16K",
+	"01-ai/Yi-1.5-34B-Chat-16K",
+	"THUDM/chatglm3-6b",
+	"deepseek-ai/DeepSeek-V2-Chat",
+	"THUDM/glm-4-9b-chat",
+	"Qwen/Qwen2-72B-Instruct",
+	"Qwen/Qwen2-7B-Instruct",
+	"Qwen/Qwen2-57B-A14B-Instruct",
+	"deepseek-ai/DeepSeek-Coder-V2-Instruct",
+	"Qwen/Qwen2-1.5B-Instruct",
+	"internlm/internlm2_5-7b-chat",
+	"BAAI/bge-large-en-v1.5",
+	"BAAI/bge-large-zh-v1.5",
+	"Pro/Qwen/Qwen2-7B-Instruct",
+	"Pro/Qwen/Qwen2-1.5B-Instruct",
+	"Pro/Qwen/Qwen1.5-7B-Chat",
+	"Pro/THUDM/glm-4-9b-chat",
+	"Pro/THUDM/chatglm3-6b",
+	"Pro/01-ai/Yi-1.5-9B-Chat-16K",
+	"Pro/01-ai/Yi-1.5-6B-Chat",
+	"Pro/google/gemma-2-9b-it",
+	"Pro/internlm/internlm2_5-7b-chat",
+	"Pro/meta-llama/Meta-Llama-3-8B-Instruct",
+	"Pro/mistralai/Mistral-7B-Instruct-v0.2",
+}

relay/adaptor/vertexai/claude/model.go

@@ -4,16 +4,16 @@ import "github.com/songquanpeng/one-api/relay/adaptor/anthropic"
 
 type Request struct {
 	// AnthropicVersion must be "vertex-2023-10-16"
-	AnthropicVersion string              `json:"anthropic_version"`
+	AnthropicVersion string `json:"anthropic_version"`
 	// Model            string              `json:"model"`
-	Messages         []anthropic.Message `json:"messages"`
-	System           string              `json:"system,omitempty"`
-	MaxTokens        int                 `json:"max_tokens,omitempty"`
-	StopSequences    []string            `json:"stop_sequences,omitempty"`
-	Stream           bool                `json:"stream,omitempty"`
-	Temperature      float64             `json:"temperature,omitempty"`
-	TopP             float64             `json:"top_p,omitempty"`
-	TopK             int                 `json:"top_k,omitempty"`
-	Tools            []anthropic.Tool    `json:"tools,omitempty"`
-	ToolChoice       any                 `json:"tool_choice,omitempty"`
+	Messages      []anthropic.Message `json:"messages"`
+	System        string              `json:"system,omitempty"`
+	MaxTokens     int                 `json:"max_tokens,omitempty"`
+	StopSequences []string            `json:"stop_sequences,omitempty"`
+	Stream        bool                `json:"stream,omitempty"`
+	Temperature   float64             `json:"temperature,omitempty"`
+	TopP          float64             `json:"top_p,omitempty"`
+	TopK          int                 `json:"top_k,omitempty"`
+	Tools         []anthropic.Tool    `json:"tools,omitempty"`
+	ToolChoice    any                 `json:"tool_choice,omitempty"`
 }

relay/adaptor/vertexai/registry.go

@@ -32,7 +32,6 @@ func init() {
 	}
 }
 
-
 type innerAIAdapter interface {
 	ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error)
 	DoResponse(c *gin.Context, resp *http.Response, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode)

relay/adaptor/vertexai/token.go

@@ -26,7 +26,6 @@ type ApplicationDefaultCredentials struct {
 	UniverseDomain          string `json:"universe_domain"`
 }
 
-
 var Cache = cache.New(50*time.Minute, 55*time.Minute)
 
 const defaultScope = "https://www.googleapis.com/auth/cloud-platform"