feat: support gpt-4-turbo-2014-04-09 vision function

CHANGELOG.md

@@ -3,7 +3,7 @@
 * 功能新增：允许为角色应用绑定模型，如指定某个角色只能使用某个模型
 * Bug修复：兼容 gpt-4-turbo-2024-04-09 模型的函数调用 Bug
 * Bug修复：修复MidJourney在任务超时后出现后面的任务覆盖前面任务的问题
-* 
+* 功能新增：支持上传图片和视觉模型。
 
 ## 4.0.2
 

api/handler/admin/api_key_handler.go

@@ -66,11 +66,6 @@ func (h *ApiKeyHandler) Save(c *gin.Context) {
 }
 
 func (h *ApiKeyHandler) List(c *gin.Context) {
-	if err := utils.CheckPermission(c, h.DB); err != nil {
-		resp.NotPermission(c)
-		return
-	}
-
 	var items []model.ApiKey
 	var keys = make([]vo.ApiKey, 0)
 	res := h.DB.Find(&items)

api/handler/admin/chat_handler.go

@@ -33,11 +33,6 @@ type chatItemVo struct {
 }
 
 func (h *ChatHandler) List(c *gin.Context) {
-	if err := utils.CheckPermission(c, h.DB); err != nil {
-		resp.NotPermission(c)
-		return
-	}
-
 	var data struct {
 		Title    string   `json:"title"`
 		UserId   uint     `json:"user_id"`

api/handler/admin/chat_model_handler.go

@@ -76,11 +76,6 @@ func (h *ChatModelHandler) Save(c *gin.Context) {
 
 // List 模型列表
 func (h *ChatModelHandler) List(c *gin.Context) {
-	if err := utils.CheckPermission(c, h.DB); err != nil {
-		resp.NotPermission(c)
-		return
-	}
-
 	session := h.DB.Session(&gorm.Session{})
 	enable := h.GetBool(c, "enable")
 	if enable {

api/handler/admin/chat_role_handler.go

@@ -51,11 +51,6 @@ func (h *ChatRoleHandler) Save(c *gin.Context) {
 }
 
 func (h *ChatRoleHandler) List(c *gin.Context) {
-	if err := utils.CheckPermission(c, h.DB); err != nil {
-		resp.NotPermission(c)
-		return
-	}
-
 	var items []model.ChatRole
 	var roles = make([]vo.ChatRole, 0)
 	res := h.DB.Order("sort_num ASC").Find(&items)

api/handler/admin/config_handler.go

@@ -70,11 +70,6 @@ func (h *ConfigHandler) Update(c *gin.Context) {
 
 // Get 获取指定的系统配置
 func (h *ConfigHandler) Get(c *gin.Context) {
-	if err := utils.CheckPermission(c, h.DB); err != nil {
-		resp.NotPermission(c)
-		return
-	}
-
 	key := c.Query("key")
 	var config model.Config
 	res := h.DB.Where("marker", key).First(&config)

api/handler/admin/function_handler.go

@@ -71,11 +71,6 @@ func (h *FunctionHandler) Set(c *gin.Context) {
 }
 
 func (h *FunctionHandler) List(c *gin.Context) {
-	if err := utils.CheckPermission(c, h.DB); err != nil {
-		resp.NotPermission(c)
-		return
-	}
-
 	var items []model.Function
 	res := h.DB.Find(&items)
 	if res.Error != nil {

api/handler/admin/order_handler.go

@@ -22,11 +22,6 @@ func NewOrderHandler(app *core.AppServer, db *gorm.DB) *OrderHandler {
 }
 
 func (h *OrderHandler) List(c *gin.Context) {
-	if err := utils.CheckPermission(c, h.DB); err != nil {
-		resp.NotPermission(c)
-		return
-	}
-
 	var data struct {
 		OrderNo  string   `json:"order_no"`
 		Status   int      `json:"status"`

api/handler/admin/reward_handler.go

@@ -21,11 +21,6 @@ func NewRewardHandler(app *core.AppServer, db *gorm.DB) *RewardHandler {
 }
 
 func (h *RewardHandler) List(c *gin.Context) {
-	if err := utils.CheckPermission(c, h.DB); err != nil {
-		resp.NotPermission(c)
-		return
-	}
-
 	var items []model.Reward
 	res := h.DB.Order("id DESC").Find(&items)
 	var rewards = make([]vo.Reward, 0)

api/handler/admin/user_handler.go

@@ -25,11 +25,6 @@ func NewUserHandler(app *core.AppServer, db *gorm.DB) *UserHandler {
 
 // List 用户列表
 func (h *UserHandler) List(c *gin.Context) {
-	if err := utils.CheckPermission(c, h.DB); err != nil {
-		resp.NotPermission(c)
-		return
-	}
-
 	page := h.GetInt(c, "page", 1)
 	pageSize := h.GetInt(c, "page_size", 20)
 	username := h.GetTrim(c, "username")

api/handler/chatimpl/chat_handler.go

@@ -339,6 +339,34 @@ func (h *ChatHandler) sendMessage(ctx context.Context, session *types.ChatSessio
 		if len(reqMgs) > 0 {
 			req.Input["messages"] = reqMgs
 		}
+	} else if session.Model.Platform == types.OpenAI { // extract image for gpt-vision model
+		imgURLs := utils.ExtractImgURL(prompt)
+		logger.Debugf("detected IMG: %+v", imgURLs)
+		var content interface{}
+		if len(imgURLs) > 0 {
+			data := make([]interface{}, 0)
+			text := prompt
+			for _, v := range imgURLs {
+				text = strings.Replace(text, v, "", 1)
+				data = append(data, gin.H{
+					"type": "image_url",
+					"image_url": gin.H{
+						"url": v,
+					},
+				})
+			}
+			data = append(data, gin.H{
+				"type": "text",
+				"text": text,
+			})
+			content = data
+		} else {
+			content = prompt
+		}
+		req.Messages = append(reqMgs, map[string]interface{}{
+			"role":    "user",
+			"content": content,
+		})
 	} else {
 		req.Messages = append(reqMgs, map[string]interface{}{
 			"role":    "user",
@@ -346,6 +374,8 @@ func (h *ChatHandler) sendMessage(ctx context.Context, session *types.ChatSessio
 		})
 	}
 
+	logger.Debugf("%+v", req.Messages)
+
 	switch session.Model.Platform {
 	case types.Azure:
 		return h.sendAzureMessage(chatCtx, req, userVo, ctx, session, role, prompt, ws)

api/test/test.go

@@ -1,12 +1,12 @@
 package main
 
 import (
+	"chatplus/utils"
 	"fmt"
-	"reflect"
 )
 
 func main() {
-	text := 1
+	text := "https://nk.img.r9it.com/chatgpt-plus/1712709360012445.png 请简单描述一下这幅图上的内容 "
-	bytes := reflect.ValueOf(text).Bytes()
+	imgURL := utils.ExtractImgURL(text)
-	fmt.Println(bytes)
+	fmt.Println(imgURL)
 }

api/utils/permission.go

@@ -1,36 +0,0 @@
-package utils
-
-import (
-	"github.com/gin-gonic/gin"
-	"gorm.io/gorm"
-)
-
-// CheckPermission Todo: 放在缓存
-// CheckPermission 检查权限
-func CheckPermission(c *gin.Context, db *gorm.DB) error {
-	return nil
-	//u, err := url.Parse(c.Request.RequestURI)
-	//if err != nil {
-	//	panic(err)
-	//}
-	//slug := strings.Replace(u.Path, "/", "_", -1)[1:]
-	//
-	//// 用户名
-	//userName, _ := c.Get(types.LoginUserID)
-	//
-	//var manager model.AdminUser
-	//db.Table("chatgpt_admin_users").Select("chatgpt_admin_users.id").Where("username = ?", userName).First(&manager)
-	//
-	//// 超级管理员不判断
-	//if manager.Id == 1 {
-	//	return nil
-	//}
-	//var roleIds []int
-	//var count int64
-	//db.Raw("SELECT `chatgpt_admin_user_roles`.role_id FROM `chatgpt_admin_users` LEFT JOIN `chatgpt_admin_user_roles` ON ( `chatgpt_admin_users`.id = `chatgpt_admin_user_roles`.admin_id ) WHERE `chatgpt_admin_users`.id = ?", manager.Id).Find(&roleIds)
-	//db.Raw("SELECT `chatgpt_admin_permissions`.slug FROM `chatgpt_admin_permissions` LEFT JOIN `chatgpt_admin_role_permissions` ON (`chatgpt_admin_permissions`.id = `chatgpt_admin_role_permissions`.permission_id) WHERE `chatgpt_admin_role_permissions`.role_id IN ? and `chatgpt_admin_permissions`.slug = ? ", roleIds, slug).Count(&count)
-	//if count > 0 {
-	//	return nil
-	//}
-	//return fmt.Errorf("没有权限")
-}

api/utils/upload.go

@@ -7,6 +7,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 )
@@ -79,3 +80,15 @@ func GetImgExt(filename string) string {
 	}
 	return ext
 }
+
+func ExtractImgURL(text string) []string {
+	re := regexp.MustCompile(`(http[s]?:\/\/.*?\.(?:png|jpg|jpeg|gif))`)
+	matches := re.FindAllStringSubmatch(text, 10)
+	urls := make([]string, 0)
+	if len(matches) > 0 {
+		for _, m := range matches {
+			urls = append(urls, m[1])
+		}
+	}
+	return urls
+}