From ac527851a0db2dbe0676f53203e80adb78ccbe23 Mon Sep 17 00:00:00 2001 From: RockYang Date: Wed, 10 Apr 2024 11:47:10 +0800 Subject: [PATCH] feat: support gpt-4-turbo-2014-04-09 vision function --- CHANGELOG.md | 2 +- api/handler/admin/api_key_handler.go | 5 ---- api/handler/admin/chat_handler.go | 5 ---- api/handler/admin/chat_model_handler.go | 5 ---- api/handler/admin/chat_role_handler.go | 5 ---- api/handler/admin/config_handler.go | 5 ---- api/handler/admin/function_handler.go | 5 ---- api/handler/admin/order_handler.go | 5 ---- api/handler/admin/reward_handler.go | 5 ---- api/handler/admin/user_handler.go | 5 ---- api/handler/chatimpl/chat_handler.go | 30 +++++++++++++++++++++ api/test/test.go | 8 +++--- api/utils/permission.go | 36 ------------------------- api/utils/upload.go | 13 +++++++++ 14 files changed, 48 insertions(+), 86 deletions(-) delete mode 100644 api/utils/permission.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ec9f7e1..7171dda7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ * 功能新增：允许为角色应用绑定模型，如指定某个角色只能使用某个模型 * Bug修复：兼容 gpt-4-turbo-2024-04-09 模型的函数调用 Bug * Bug修复：修复MidJourney在任务超时后出现后面的任务覆盖前面任务的问题 -* +* 功能新增：支持上传图片和视觉模型。 ## 4.0.2 diff --git a/api/handler/admin/api_key_handler.go b/api/handler/admin/api_key_handler.go index 0d3873b9..5566b0c0 100644 --- a/api/handler/admin/api_key_handler.go +++ b/api/handler/admin/api_key_handler.go @@ -66,11 +66,6 @@ func (h *ApiKeyHandler) Save(c *gin.Context) { } func (h *ApiKeyHandler) List(c *gin.Context) { - if err := utils.CheckPermission(c, h.DB); err != nil { - resp.NotPermission(c) - return - } - var items []model.ApiKey var keys = make([]vo.ApiKey, 0) res := h.DB.Find(&items) diff --git a/api/handler/admin/chat_handler.go b/api/handler/admin/chat_handler.go index 3d29d165..64fb5587 100644 --- a/api/handler/admin/chat_handler.go +++ b/api/handler/admin/chat_handler.go @@ -33,11 +33,6 @@ type chatItemVo struct { } func (h *ChatHandler) List(c *gin.Context) { - if err := utils.CheckPermission(c, h.DB); err != nil { - resp.NotPermission(c) - return - } - var data struct { Title string `json:"title"` UserId uint `json:"user_id"` diff --git a/api/handler/admin/chat_model_handler.go b/api/handler/admin/chat_model_handler.go index 47113055..4f6ee23e 100644 --- a/api/handler/admin/chat_model_handler.go +++ b/api/handler/admin/chat_model_handler.go @@ -76,11 +76,6 @@ func (h *ChatModelHandler) Save(c *gin.Context) { // List 模型列表 func (h *ChatModelHandler) List(c *gin.Context) { - if err := utils.CheckPermission(c, h.DB); err != nil { - resp.NotPermission(c) - return - } - session := h.DB.Session(&gorm.Session{}) enable := h.GetBool(c, "enable") if enable { diff --git a/api/handler/admin/chat_role_handler.go b/api/handler/admin/chat_role_handler.go index 4d119faf..caec61b9 100644 --- a/api/handler/admin/chat_role_handler.go +++ b/api/handler/admin/chat_role_handler.go @@ -51,11 +51,6 @@ func (h *ChatRoleHandler) Save(c *gin.Context) { } func (h *ChatRoleHandler) List(c *gin.Context) { - if err := utils.CheckPermission(c, h.DB); err != nil { - resp.NotPermission(c) - return - } - var items []model.ChatRole var roles = make([]vo.ChatRole, 0) res := h.DB.Order("sort_num ASC").Find(&items) diff --git a/api/handler/admin/config_handler.go b/api/handler/admin/config_handler.go index 7ad863aa..d8130354 100644 --- a/api/handler/admin/config_handler.go +++ b/api/handler/admin/config_handler.go @@ -70,11 +70,6 @@ func (h *ConfigHandler) Update(c *gin.Context) { // Get 获取指定的系统配置 func (h *ConfigHandler) Get(c *gin.Context) { - if err := utils.CheckPermission(c, h.DB); err != nil { - resp.NotPermission(c) - return - } - key := c.Query("key") var config model.Config res := h.DB.Where("marker", key).First(&config) diff --git a/api/handler/admin/function_handler.go b/api/handler/admin/function_handler.go index 97940931..d9eed1fc 100644 --- a/api/handler/admin/function_handler.go +++ b/api/handler/admin/function_handler.go @@ -71,11 +71,6 @@ func (h *FunctionHandler) Set(c *gin.Context) { } func (h *FunctionHandler) List(c *gin.Context) { - if err := utils.CheckPermission(c, h.DB); err != nil { - resp.NotPermission(c) - return - } - var items []model.Function res := h.DB.Find(&items) if res.Error != nil { diff --git a/api/handler/admin/order_handler.go b/api/handler/admin/order_handler.go index 993b3995..1183e01f 100644 --- a/api/handler/admin/order_handler.go +++ b/api/handler/admin/order_handler.go @@ -22,11 +22,6 @@ func NewOrderHandler(app *core.AppServer, db *gorm.DB) *OrderHandler { } func (h *OrderHandler) List(c *gin.Context) { - if err := utils.CheckPermission(c, h.DB); err != nil { - resp.NotPermission(c) - return - } - var data struct { OrderNo string `json:"order_no"` Status int `json:"status"` diff --git a/api/handler/admin/reward_handler.go b/api/handler/admin/reward_handler.go index e2d283e3..a2c44cb9 100644 --- a/api/handler/admin/reward_handler.go +++ b/api/handler/admin/reward_handler.go @@ -21,11 +21,6 @@ func NewRewardHandler(app *core.AppServer, db *gorm.DB) *RewardHandler { } func (h *RewardHandler) List(c *gin.Context) { - if err := utils.CheckPermission(c, h.DB); err != nil { - resp.NotPermission(c) - return - } - var items []model.Reward res := h.DB.Order("id DESC").Find(&items) var rewards = make([]vo.Reward, 0) diff --git a/api/handler/admin/user_handler.go b/api/handler/admin/user_handler.go index 1bc70b40..430b66bb 100644 --- a/api/handler/admin/user_handler.go +++ b/api/handler/admin/user_handler.go @@ -25,11 +25,6 @@ func NewUserHandler(app *core.AppServer, db *gorm.DB) *UserHandler { // List 用户列表 func (h *UserHandler) List(c *gin.Context) { - if err := utils.CheckPermission(c, h.DB); err != nil { - resp.NotPermission(c) - return - } - page := h.GetInt(c, "page", 1) pageSize := h.GetInt(c, "page_size", 20) username := h.GetTrim(c, "username") diff --git a/api/handler/chatimpl/chat_handler.go b/api/handler/chatimpl/chat_handler.go index 3a4cb91c..5da9af69 100644 --- a/api/handler/chatimpl/chat_handler.go +++ b/api/handler/chatimpl/chat_handler.go @@ -339,6 +339,34 @@ func (h *ChatHandler) sendMessage(ctx context.Context, session *types.ChatSessio if len(reqMgs) > 0 { req.Input["messages"] = reqMgs } + } else if session.Model.Platform == types.OpenAI { // extract image for gpt-vision model + imgURLs := utils.ExtractImgURL(prompt) + logger.Debugf("detected IMG: %+v", imgURLs) + var content interface{} + if len(imgURLs) > 0 { + data := make([]interface{}, 0) + text := prompt + for _, v := range imgURLs { + text = strings.Replace(text, v, "", 1) + data = append(data, gin.H{ + "type": "image_url", + "image_url": gin.H{ + "url": v, + }, + }) + } + data = append(data, gin.H{ + "type": "text", + "text": text, + }) + content = data + } else { + content = prompt + } + req.Messages = append(reqMgs, map[string]interface{}{ + "role": "user", + "content": content, + }) } else { req.Messages = append(reqMgs, map[string]interface{}{ "role": "user", @@ -346,6 +374,8 @@ func (h *ChatHandler) sendMessage(ctx context.Context, session *types.ChatSessio }) } + logger.Debugf("%+v", req.Messages) + switch session.Model.Platform { case types.Azure: return h.sendAzureMessage(chatCtx, req, userVo, ctx, session, role, prompt, ws) diff --git a/api/test/test.go b/api/test/test.go index cc826def..ff31c2f5 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -1,12 +1,12 @@ package main import ( + "chatplus/utils" "fmt" - "reflect" ) func main() { - text := 1 - bytes := reflect.ValueOf(text).Bytes() - fmt.Println(bytes) + text := "https://nk.img.r9it.com/chatgpt-plus/1712709360012445.png 请简单描述一下这幅图上的内容 " + imgURL := utils.ExtractImgURL(text) + fmt.Println(imgURL) } diff --git a/api/utils/permission.go b/api/utils/permission.go deleted file mode 100644 index dfbee3de..00000000 --- a/api/utils/permission.go +++ /dev/null @@ -1,36 +0,0 @@ -package utils - -import ( - "github.com/gin-gonic/gin" - "gorm.io/gorm" -) - -// CheckPermission Todo: 放在缓存 -// CheckPermission 检查权限 -func CheckPermission(c *gin.Context, db *gorm.DB) error { - return nil - //u, err := url.Parse(c.Request.RequestURI) - //if err != nil { - // panic(err) - //} - //slug := strings.Replace(u.Path, "/", "_", -1)[1:] - // - //// 用户名 - //userName, _ := c.Get(types.LoginUserID) - // - //var manager model.AdminUser - //db.Table("chatgpt_admin_users").Select("chatgpt_admin_users.id").Where("username = ?", userName).First(&manager) - // - //// 超级管理员不判断 - //if manager.Id == 1 { - // return nil - //} - //var roleIds []int - //var count int64 - //db.Raw("SELECT `chatgpt_admin_user_roles`.role_id FROM `chatgpt_admin_users` LEFT JOIN `chatgpt_admin_user_roles` ON ( `chatgpt_admin_users`.id = `chatgpt_admin_user_roles`.admin_id ) WHERE `chatgpt_admin_users`.id = ?", manager.Id).Find(&roleIds) - //db.Raw("SELECT `chatgpt_admin_permissions`.slug FROM `chatgpt_admin_permissions` LEFT JOIN `chatgpt_admin_role_permissions` ON (`chatgpt_admin_permissions`.id = `chatgpt_admin_role_permissions`.permission_id) WHERE `chatgpt_admin_role_permissions`.role_id IN ? and `chatgpt_admin_permissions`.slug = ? ", roleIds, slug).Count(&count) - //if count > 0 { - // return nil - //} - //return fmt.Errorf("没有权限") -} diff --git a/api/utils/upload.go b/api/utils/upload.go index 1bab2aca..695d9183 100644 --- a/api/utils/upload.go +++ b/api/utils/upload.go @@ -7,6 +7,7 @@ import ( "net/url" "os" "path/filepath" + "regexp" "strings" "time" ) @@ -79,3 +80,15 @@ func GetImgExt(filename string) string { } return ext } + +func ExtractImgURL(text string) []string { + re := regexp.MustCompile(`(http[s]?:\/\/.*?\.(?:png|jpg|jpeg|gif))`) + matches := re.FindAllStringSubmatch(text, 10) + urls := make([]string, 0) + if len(matches) > 0 { + for _, m := range matches { + urls = append(urls, m[1]) + } + } + return urls +}