support gemini-pro-vision

controller/model.go

@@ -432,6 +432,15 @@ func init() {
 			Root:       "gemini-pro",
 			Parent:     nil,
 		},
+		{
+			Id:         "gemini-pro-vision",
+			Object:     "model",
+			Created:    1677649963,
+			OwnedBy:    "google",
+			Permission: permission,
+			Root:       "gemini-pro-vision",
+			Parent:     nil,
+		},
 		{
 			Id:         "chatglm_turbo",
 			Object:     "model",

controller/relay-gemini.go

@@ -12,6 +12,10 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
+const (
+	GeminiVisionMaxImageNum = 16
+)
+
 type GeminiChatRequest struct {
 	Contents         []GeminiChatContent        `json:"contents"`
 	SafetySettings   []GeminiChatSafetySettings `json:"safety_settings,omitempty"`
@@ -97,6 +101,31 @@ func requestOpenAI2Gemini(textRequest GeneralOpenAIRequest) *GeminiChatRequest {
 				},
 			},
 		}
+		openaiContent := message.ParseContent()
+		var parts []GeminiPart
+		imageNum := 0
+		for _, part := range openaiContent {
+
+			if part.Type == ContentTypeText {
+				parts = append(parts, GeminiPart{
+					Text: part.Text,
+				})
+			} else if part.Type == ContentTypeImageURL {
+				imageNum += 1
+				if imageNum > GeminiVisionMaxImageNum {
+					continue
+				}
+				mimeType, data, _ := common.GetImageFromUrl(part.ImageUrl.(MessageImageUrl).Url)
+				parts = append(parts, GeminiPart{
+					InlineData: &GeminiInlineData{
+						MimeType: mimeType,
+						Data:     data,
+					},
+				})
+			}
+		}
+		content.Parts = parts
+
 		// there's no assistant role in gemini and API shall vomit if Role is not user or model
 		if content.Role == "assistant" {
 			content.Role = "model"

controller/relay-utils.go

@@ -76,12 +76,13 @@ func getImageToken(imageUrl *MessageImageUrl) (int, error) {
 	}
 	var config image.Config
 	var err error
+	var format string
 	if strings.HasPrefix(imageUrl.Url, "http") {
 		common.SysLog(fmt.Sprintf("downloading image: %s", imageUrl.Url))
-		config, err = common.DecodeUrlImageData(imageUrl.Url)
+		config, format, err = common.DecodeUrlImageData(imageUrl.Url)
 	} else {
 		common.SysLog(fmt.Sprintf("decoding image"))
-		config, err = common.DecodeBase64ImageData(imageUrl.Url)
+		config, format, err = common.DecodeBase64ImageData(imageUrl.Url)
 	}
 	if err != nil {
 		return 0, err
@@ -101,7 +102,7 @@ func getImageToken(imageUrl *MessageImageUrl) (int, error) {
 
 	shortSide := config.Width
 	otherSide := config.Height
-	log.Printf("width: %d, height: %d", config.Width, config.Height)
+	log.Printf("format: %s, width: %d, height: %d", format, config.Width, config.Height)
 	// 缩放倍数
 	scale := 1.0
 	if config.Height < shortSide {

controller/relay.go

@@ -29,6 +29,60 @@ type MessageImageUrl struct {
 	Detail string `json:"detail"`
 }
 
+const (
+	ContentTypeText     = "text"
+	ContentTypeImageURL = "image_url"
+)
+
+func (m Message) ParseContent() []MediaMessage {
+	var contentList []MediaMessage
+	var stringContent string
+	if err := json.Unmarshal(m.Content, &stringContent); err == nil {
+		contentList = append(contentList, MediaMessage{
+			Type: ContentTypeText,
+			Text: stringContent,
+		})
+		return contentList
+	}
+	var arrayContent []json.RawMessage
+	if err := json.Unmarshal(m.Content, &arrayContent); err == nil {
+		for _, contentItem := range arrayContent {
+			var contentMap map[string]any
+			if err := json.Unmarshal(contentItem, &contentMap); err != nil {
+				continue
+			}
+			switch contentMap["type"] {
+			case ContentTypeText:
+				if subStr, ok := contentMap["text"].(string); ok {
+					contentList = append(contentList, MediaMessage{
+						Type: ContentTypeText,
+						Text: subStr,
+					})
+				}
+			case ContentTypeImageURL:
+				if subObj, ok := contentMap["image_url"].(map[string]any); ok {
+					detail, ok := subObj["detail"]
+					if ok {
+						subObj["detail"] = detail.(string)
+					} else {
+						subObj["detail"] = "auto"
+					}
+					contentList = append(contentList, MediaMessage{
+						Type: ContentTypeImageURL,
+						ImageUrl: MessageImageUrl{
+							Url:    subObj["url"].(string),
+							Detail: subObj["detail"].(string),
+						},
+					})
+				}
+			}
+		}
+		return contentList
+	}
+
+	return nil
+}
+
 const (
 	RelayModeUnknown = iota
 	RelayModeChatCompletions