From 3271b924fa9d8fa665b70654fde77653a8247600 Mon Sep 17 00:00:00 2001 From: GeekMaster Date: Mon, 1 Sep 2025 17:41:45 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=87=E6=9C=AC=E5=AE=A1=E6=A0=B8=E8=AE=B0?= =?UTF-8?q?=E5=BD=95=E5=8A=9F=E8=83=BD=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/core/types/moderation.go | 17 + api/handler/admin/moderation_handler.go | 231 ++++++++ api/handler/chat_handler.go | 69 ++- api/handler/dalle_handler.go | 40 +- api/handler/jimeng_handler.go | 40 +- api/handler/mj_handler.go | 44 +- api/handler/sd_handler.go | 51 +- api/handler/suno_handler.go | 40 +- api/handler/video_handler.go | 40 +- api/main.go | 4 + api/store/model/moderation.go | 3 +- web/src/assets/css/sd-task-dialog.scss | 4 +- web/src/components/admin/AdminSidebar.vue | 5 + web/src/router.js | 8 +- .../ModerationConfig.vue | 0 .../views/admin/moderation/ModerationList.vue | 499 ++++++++++++++++++ 16 files changed, 1028 insertions(+), 67 deletions(-) create mode 100644 api/handler/admin/moderation_handler.go rename web/src/views/admin/{settings => moderation}/ModerationConfig.vue (100%) create mode 100644 web/src/views/admin/moderation/ModerationList.vue diff --git a/api/core/types/moderation.go b/api/core/types/moderation.go index 20a9eb94..a5310439 100644 --- a/api/core/types/moderation.go +++ b/api/core/types/moderation.go @@ -53,4 +53,21 @@ var ModerationCategories = map[string]string{ "porn": "明确的色情内容", "insult": "具有侮辱、攻击性语言、人身攻击或冒犯性表达", "violence": "包含暴力、血腥、攻击行为或煽动暴力的言论", + "illegal": "涉及违法活动的内容，如诈骗、赌博等", + "terror": "宣扬恐怖主义、极端暴力或煽动恐怖行为的内容", + "ad": "垃圾广告或未经许可的推广内容", + "spam": "无意义重复内容或诱导性信息", + "abuse": "人身攻击、恶意辱骂或侮辱性言论", + "polity": "涉及国家政治、领导人或政策的违规讨论内容", } + +// 敏感词来源 +const ( + ModerationSourceChat = "chat" + ModerationSourceMJ = "mj" + ModerationSourceDalle = "dalle" + ModerationSourceSD = "sd" + ModerationSourceSuno = "suno" + ModerationSourceVideo = "video" + ModerationSourceJiMeng = "jimeng" +) diff --git a/api/handler/admin/moderation_handler.go b/api/handler/admin/moderation_handler.go new file mode 100644 index 00000000..55cf25ce --- /dev/null +++ b/api/handler/admin/moderation_handler.go @@ -0,0 +1,231 @@ +package admin + +// * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +// * Copyright 2023 The Geek-AI Authors. All rights reserved. +// * Use of this source code is governed by a Apache-2.0 license +// * that can be found in the LICENSE file. +// * @Author yangjian102621@163.com +// * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import ( + "geekai/core" + "geekai/core/middleware" + "geekai/core/types" + "geekai/handler" + "geekai/store/model" + "geekai/utils" + "geekai/utils/resp" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" +) + +type ModerationHandler struct { + handler.BaseHandler +} + +func NewModerationHandler(app *core.AppServer, db *gorm.DB) *ModerationHandler { + return &ModerationHandler{BaseHandler: handler.BaseHandler{DB: db, App: app}} +} + +// RegisterRoutes 注册路由 +func (h *ModerationHandler) RegisterRoutes() { + group := h.App.Engine.Group("/api/admin/moderation/") + + // 需要管理员授权的接口 + group.Use(middleware.AdminAuthMiddleware(h.App.Config.AdminSession.SecretKey, h.App.Redis)) + { + group.POST("list", h.List) + group.GET("remove", h.Remove) + group.POST("batch-remove", h.BatchRemove) + group.GET("source-list", h.GetSourceList) + } +} + +// List 获取文本审核记录列表 +func (h *ModerationHandler) List(c *gin.Context) { + var data struct { + Username string `json:"username"` + Source string `json:"source"` + StartDate string `json:"start_date"` + EndDate string `json:"end_date"` + Page int `json:"page"` + PageSize int `json:"page_size"` + } + + if err := c.ShouldBindJSON(&data); err != nil { + resp.ERROR(c, types.InvalidArgs) + return + } + + session := h.DB.Session(&gorm.Session{}) + + // 构建查询条件 + if data.Username != "" { + // 通过用户名查找用户ID + var user model.User + if err := h.DB.Where("username LIKE ?", "%"+data.Username+"%").First(&user).Error; err == nil { + session = session.Where("user_id", user.Id) + } + } + + if data.Source != "" { + session = session.Where("source", data.Source) + } + + if data.StartDate != "" && data.EndDate != "" { + startTime := data.StartDate + " 00:00:00" + endTime := data.EndDate + " 23:59:59" + session = session.Where("created_at >= ? AND created_at <= ?", startTime, endTime) + } + + // 统计总数 + var total int64 + session.Model(&model.Moderation{}).Count(&total) + + // 分页 + page := data.Page + pageSize := data.PageSize + if page <= 0 { + page = 1 + } + if pageSize <= 0 { + pageSize = 20 + } + + offset := (page - 1) * pageSize + session = session.Offset(offset).Limit(pageSize) + + // 查询数据 + var items []model.Moderation + err := session.Order("id DESC").Find(&items).Error + if err != nil { + resp.ERROR(c, err.Error()) + return + } + + // 获取用户信息 + userIds := make([]uint, 0) + for _, item := range items { + userIds = append(userIds, item.UserId) + } + + var users []model.User + if len(userIds) > 0 { + h.DB.Where("id IN ?", userIds).Find(&users) + } + + userMap := make(map[uint]string) + for _, user := range users { + userMap[user.Id] = user.Username + } + + // 转换为响应数据 + list := make([]map[string]any, 0) + for _, item := range items { + var moderation types.ModerationResult + err := utils.JsonDecode(item.Result, &moderation) + if err != nil { + continue + } + var result []string + for value, label := range types.ModerationCategories { + if moderation.Categories[value] { + result = append(result, label) + } + } + list = append(list, map[string]any{ + "id": item.Id, + "user_id": item.UserId, + "username": userMap[item.UserId], + "source": item.Source, + "input": item.Input, + "output": item.Output, + "result": result, + "created_at": item.CreatedAt.Unix(), + }) + } + + resp.SUCCESS(c, map[string]any{ + "items": list, + "total": total, + "page": page, + "page_size": pageSize, + }) +} + +func (h *ModerationHandler) Remove(c *gin.Context) { + id := h.GetInt(c, "id", 0) + if id <= 0 { + resp.ERROR(c, types.InvalidArgs) + return + } + + err := h.DB.Where("id", id).Delete(&model.Moderation{}).Error + if err != nil { + resp.ERROR(c, err.Error()) + return + } + resp.SUCCESS(c) +} + +// BatchRemove 批量删除文本审核记录 +func (h *ModerationHandler) BatchRemove(c *gin.Context) { + var data struct { + Ids []uint `json:"ids"` + } + + if err := c.ShouldBindJSON(&data); err != nil { + resp.ERROR(c, types.InvalidArgs) + return + } + + if len(data.Ids) == 0 { + resp.ERROR(c, "请选择要删除的记录") + return + } + + err := h.DB.Where("id IN ?", data.Ids).Delete(&model.Moderation{}).Error + if err != nil { + resp.ERROR(c, err.Error()) + return + } + + resp.SUCCESS(c) +} + +// 获取 source 列表 +func (h *ModerationHandler) GetSourceList(c *gin.Context) { + sources := []gin.H{ + { + "id": types.ModerationSourceChat, + "name": "AI对话", + }, + { + "id": types.ModerationSourceMJ, + "name": "Midjourney 绘图", + }, + { + "id": types.ModerationSourceDalle, + "name": "Dalle 绘图", + }, + { + "id": types.ModerationSourceSD, + "name": "StableDiffusion 绘图", + }, + { + "id": types.ModerationSourceSuno, + "name": "Suno 音乐", + }, + { + "id": types.ModerationSourceVideo, + "name": "视频生成", + }, + { + "id": types.ModerationSourceJiMeng, + "name": "即梦AI", + }, + } + + resp.SUCCESS(c, sources) +} diff --git a/api/handler/chat_handler.go b/api/handler/chat_handler.go index d19b9528..9785a185 100644 --- a/api/handler/chat_handler.go +++ b/api/handler/chat_handler.go @@ -17,6 +17,7 @@ import ( "geekai/core/middleware" "geekai/core/types" "geekai/service" + "geekai/service/moderation" "geekai/service/oss" "geekai/store/model" "geekai/store/vo" @@ -62,21 +63,23 @@ type ChatInput struct { type ChatHandler struct { BaseHandler - redis *redis.Client - uploadManager *oss.UploaderManager - licenseService *service.LicenseService - ReqCancelFunc *types.LMap[string, context.CancelFunc] // HttpClient 请求取消 handle function - userService *service.UserService + redis *redis.Client + uploadManager *oss.UploaderManager + licenseService *service.LicenseService + ReqCancelFunc *types.LMap[string, context.CancelFunc] // HttpClient 请求取消 handle function + userService *service.UserService + moderationManager *moderation.ServiceManager } -func NewChatHandler(app *core.AppServer, db *gorm.DB, redis *redis.Client, manager *oss.UploaderManager, licenseService *service.LicenseService, userService *service.UserService) *ChatHandler { +func NewChatHandler(app *core.AppServer, db *gorm.DB, redis *redis.Client, manager *oss.UploaderManager, licenseService *service.LicenseService, userService *service.UserService, moderationManager *moderation.ServiceManager) *ChatHandler { return &ChatHandler{ - BaseHandler: BaseHandler{App: app, DB: db}, - redis: redis, - uploadManager: manager, - licenseService: licenseService, - ReqCancelFunc: types.NewLMap[string, context.CancelFunc](), - userService: userService, + BaseHandler: BaseHandler{App: app, DB: db}, + redis: redis, + uploadManager: manager, + licenseService: licenseService, + ReqCancelFunc: types.NewLMap[string, context.CancelFunc](), + userService: userService, + moderationManager: moderationManager, } } @@ -309,6 +312,14 @@ func (h *ChatHandler) sendMessage(ctx context.Context, input ChatInput, c *gin.C } reqMgs := make([]any, 0) + // 添加引导提示词,防止模型生成违规内容 + if h.App.SysConfig.Moderation.EnableGuide { + reqMgs = append(reqMgs, map[string]any{ + "role": "system", + "content": h.App.SysConfig.Moderation.GuidePrompt, + }) + } + for i := len(chatCtx) - 1; i >= 0; i-- { reqMgs = append(reqMgs, chatCtx[i]) } @@ -352,16 +363,16 @@ func (h *ChatHandler) sendMessage(ctx context.Context, input ChatInput, c *gin.C } if len(imgList) > 0 { - imgList = append(imgList, map[string]interface{}{ + imgList = append(imgList, map[string]any{ "type": "text", "text": input.Prompt, }) - req.Messages = append(reqMgs, map[string]interface{}{ + req.Messages = append(reqMgs, map[string]any{ "role": "user", "content": imgList, }) } else { - req.Messages = append(reqMgs, map[string]interface{}{ + req.Messages = append(reqMgs, map[string]any{ "role": "user", "content": finalPrompt, }) @@ -557,6 +568,34 @@ func (h *ChatHandler) saveChatHistory( promptCreatedAt time.Time, replyCreatedAt time.Time) { + // 文本审核 + if h.App.SysConfig.Moderation.Enable { + moderationResult, err := h.moderationManager.GetService().Moderate(usage.Content) + if err != nil { + logger.Error("failed to moderate content: ", err) + } + logger.Debugf("moderationResult: %+v", moderationResult) + if moderationResult.Flagged { + // 记录违规内容 + moderation := model.Moderation{ + UserId: userVo.Id, + Source: types.ModerationSourceChat, + Input: usage.Prompt, + Output: usage.Content, + Result: utils.JsonEncode(moderationResult), + } + err = h.DB.Create(&moderation).Error + if err != nil { + logger.Error("failed to save moderation: ", err) + } + pushMessage(c, ChatEventError, "很抱歉，内容触发敏感词预警，AI 无法回答！！！") + // 更新用户算力 + if input.ChatModel.Power > 0 { + h.subUserPower(userVo, input, 0, 0) + } + return + } + } // 追加聊天记录 // for prompt var promptTokens, replyTokens, totalTokens int diff --git a/api/handler/dalle_handler.go b/api/handler/dalle_handler.go index 5d955c96..959b3298 100644 --- a/api/handler/dalle_handler.go +++ b/api/handler/dalle_handler.go @@ -14,6 +14,7 @@ import ( "geekai/core/types" "geekai/service" "geekai/service/dalle" + "geekai/service/moderation" "geekai/service/oss" "geekai/store/model" "geekai/store/vo" @@ -26,16 +27,18 @@ import ( type DallJobHandler struct { BaseHandler - dallService *dalle.Service - uploader *oss.UploaderManager - userService *service.UserService + dallService *dalle.Service + uploader *oss.UploaderManager + userService *service.UserService + moderationManager *moderation.ServiceManager } -func NewDallJobHandler(app *core.AppServer, db *gorm.DB, service *dalle.Service, manager *oss.UploaderManager, userService *service.UserService) *DallJobHandler { +func NewDallJobHandler(app *core.AppServer, db *gorm.DB, service *dalle.Service, manager *oss.UploaderManager, userService *service.UserService, moderationManager *moderation.ServiceManager) *DallJobHandler { return &DallJobHandler{ - dallService: service, - uploader: manager, - userService: userService, + dallService: service, + uploader: manager, + userService: userService, + moderationManager: moderationManager, BaseHandler: BaseHandler{ App: app, DB: db, @@ -69,6 +72,29 @@ func (h *DallJobHandler) Image(c *gin.Context) { return } + // 文本审核 + if h.App.SysConfig.Moderation.Enable { + moderationResult, err := h.moderationManager.GetService().Moderate(data.Prompt) + if err != nil { + logger.Error("failed to moderate content: ", err) + } + if moderationResult.Flagged { + // 记录违规内容 + moderation := model.Moderation{ + UserId: h.GetLoginUserId(c), + Source: types.ModerationSourceDalle, + Input: data.Prompt, + Result: utils.JsonEncode(moderationResult), + } + err = h.DB.Create(&moderation).Error + if err != nil { + logger.Error("failed to save moderation: ", err) + } + resp.ERROR(c, "当前创作内容包含敏感词，提示词未通过文本审核，请重新输入！") + return + } + } + var chatModel model.ChatModel if res := h.DB.Where("id = ?", data.ModelId).First(&chatModel); res.Error != nil { resp.ERROR(c, "模型不存在") diff --git a/api/handler/jimeng_handler.go b/api/handler/jimeng_handler.go index 22787f12..b2b9670e 100644 --- a/api/handler/jimeng_handler.go +++ b/api/handler/jimeng_handler.go @@ -7,6 +7,7 @@ import ( "geekai/core/types" "geekai/service" "geekai/service/jimeng" + "geekai/service/moderation" "geekai/store/model" "geekai/store/vo" "geekai/utils" @@ -19,16 +20,18 @@ import ( // JimengHandler 即梦AI处理器 type JimengHandler struct { BaseHandler - jimengService *jimeng.Service - userService *service.UserService + jimengService *jimeng.Service + userService *service.UserService + moderationManager *moderation.ServiceManager } // NewJimengHandler 创建即梦AI处理器 -func NewJimengHandler(app *core.AppServer, jimengService *jimeng.Service, db *gorm.DB, userService *service.UserService) *JimengHandler { +func NewJimengHandler(app *core.AppServer, jimengService *jimeng.Service, db *gorm.DB, userService *service.UserService, moderationManager *moderation.ServiceManager) *JimengHandler { return &JimengHandler{ - BaseHandler: BaseHandler{App: app, DB: db}, - jimengService: jimengService, - userService: userService, + BaseHandler: BaseHandler{App: app, DB: db}, + jimengService: jimengService, + userService: userService, + moderationManager: moderationManager, } } @@ -75,6 +78,31 @@ func (h *JimengHandler) CreateTask(c *gin.Context) { resp.ERROR(c, types.InvalidArgs) return } + + // 文本审核 + if h.App.SysConfig.Moderation.Enable { + moderationResult, err := h.moderationManager.GetService().Moderate(req.Prompt) + if err != nil { + logger.Error("failed to moderate content: ", err) + } + if moderationResult.Flagged { + // 记录违规内容 + moderation := model.Moderation{ + UserId: h.GetLoginUserId(c), + Source: types.ModerationSourceJiMeng, + Input: req.Prompt, + Result: utils.JsonEncode(moderationResult), + } + err = h.DB.Create(&moderation).Error + if err != nil { + logger.Error("failed to save moderation: ", err) + } + resp.ERROR(c, "当前创作内容包含敏感词，请重新输入！") + return + } + + } + // 新增：除图像特效外，其他任务类型必须有提示词 if req.TaskType != "image_effects" && req.Prompt == "" { resp.ERROR(c, "提示词不能为空") diff --git a/api/handler/mj_handler.go b/api/handler/mj_handler.go index c2075a20..245dc993 100644 --- a/api/handler/mj_handler.go +++ b/api/handler/mj_handler.go @@ -14,6 +14,7 @@ import ( "geekai/core/types" "geekai/service" "geekai/service/mj" + "geekai/service/moderation" "geekai/service/oss" "geekai/store/model" "geekai/store/vo" @@ -28,18 +29,20 @@ import ( type MidJourneyHandler struct { BaseHandler - mjService *mj.Service - snowflake *service.Snowflake - uploader *oss.UploaderManager - userService *service.UserService + mjService *mj.Service + snowflake *service.Snowflake + uploader *oss.UploaderManager + userService *service.UserService + moderationManager *moderation.ServiceManager } -func NewMidJourneyHandler(app *core.AppServer, db *gorm.DB, snowflake *service.Snowflake, service *mj.Service, manager *oss.UploaderManager, userService *service.UserService) *MidJourneyHandler { +func NewMidJourneyHandler(app *core.AppServer, db *gorm.DB, snowflake *service.Snowflake, service *mj.Service, manager *oss.UploaderManager, userService *service.UserService, moderationManager *moderation.ServiceManager) *MidJourneyHandler { return &MidJourneyHandler{ - snowflake: snowflake, - mjService: service, - uploader: manager, - userService: userService, + snowflake: snowflake, + mjService: service, + uploader: manager, + userService: userService, + moderationManager: moderationManager, BaseHandler: BaseHandler{ App: app, DB: db, @@ -110,6 +113,29 @@ func (h *MidJourneyHandler) Image(c *gin.Context) { return } + // 文本审核 + if h.App.SysConfig.Moderation.Enable { + moderationResult, err := h.moderationManager.GetService().Moderate(data.Prompt) + if err != nil { + logger.Error("failed to moderate content: ", err) + } + if moderationResult.Flagged { + // 记录违规内容 + moderation := model.Moderation{ + UserId: h.GetLoginUserId(c), + Source: types.ModerationSourceMJ, + Input: data.Prompt, + Result: utils.JsonEncode(moderationResult), + } + err = h.DB.Create(&moderation).Error + if err != nil { + logger.Error("failed to save moderation: ", err) + } + resp.ERROR(c, "当前创作内容包含敏感词，请重新输入！") + return + } + } + var params = "" if data.Rate != "" && !strings.Contains(params, "--ar") { params += " --ar " + data.Rate diff --git a/api/handler/sd_handler.go b/api/handler/sd_handler.go index 808aa468..ce19f42f 100644 --- a/api/handler/sd_handler.go +++ b/api/handler/sd_handler.go @@ -13,6 +13,7 @@ import ( "geekai/core/middleware" "geekai/core/types" "geekai/service" + "geekai/service/moderation" "geekai/service/oss" "geekai/service/sd" "geekai/store" @@ -29,12 +30,13 @@ import ( type SdJobHandler struct { BaseHandler - redis *redis.Client - sdService *sd.Service - uploader *oss.UploaderManager - snowflake *service.Snowflake - leveldb *store.LevelDB - userService *service.UserService + redis *redis.Client + sdService *sd.Service + uploader *oss.UploaderManager + snowflake *service.Snowflake + leveldb *store.LevelDB + userService *service.UserService + moderationManager *moderation.ServiceManager } func NewSdJobHandler(app *core.AppServer, @@ -43,13 +45,15 @@ func NewSdJobHandler(app *core.AppServer, manager *oss.UploaderManager, snowflake *service.Snowflake, userService *service.UserService, - levelDB *store.LevelDB) *SdJobHandler { + levelDB *store.LevelDB, + moderationManager *moderation.ServiceManager) *SdJobHandler { return &SdJobHandler{ - sdService: service, - uploader: manager, - snowflake: snowflake, - leveldb: levelDB, - userService: userService, + sdService: service, + uploader: manager, + snowflake: snowflake, + leveldb: levelDB, + userService: userService, + moderationManager: moderationManager, BaseHandler: BaseHandler{ App: app, DB: db, @@ -102,6 +106,29 @@ func (h *SdJobHandler) Image(c *gin.Context) { return } + if h.App.SysConfig.Moderation.Enable { + moderationResult, err := h.moderationManager.GetService().Moderate(data.Prompt) + if err != nil { + logger.Error("failed to moderate content: ", err) + } + if moderationResult.Flagged { + // 记录违规内容 + moderation := model.Moderation{ + UserId: h.GetLoginUserId(c), + Source: types.ModerationSourceSD, + Input: data.Prompt, + Result: utils.JsonEncode(moderationResult), + } + err = h.DB.Create(&moderation).Error + if err != nil { + logger.Error("failed to save moderation: ", err) + } + resp.ERROR(c, "当前创作内容包含敏感词，请重新输入！") + return + } + + } + if data.Width <= 0 { data.Width = 512 } diff --git a/api/handler/suno_handler.go b/api/handler/suno_handler.go index 608023c7..73db2e42 100644 --- a/api/handler/suno_handler.go +++ b/api/handler/suno_handler.go @@ -13,6 +13,7 @@ import ( "geekai/core/middleware" "geekai/core/types" "geekai/service" + "geekai/service/moderation" "geekai/service/oss" "geekai/service/suno" "geekai/store/model" @@ -27,20 +28,22 @@ import ( type SunoHandler struct { BaseHandler - sunoService *suno.Service - uploader *oss.UploaderManager - userService *service.UserService + sunoService *suno.Service + uploader *oss.UploaderManager + userService *service.UserService + moderationManager *moderation.ServiceManager } -func NewSunoHandler(app *core.AppServer, db *gorm.DB, service *suno.Service, uploader *oss.UploaderManager, userService *service.UserService) *SunoHandler { +func NewSunoHandler(app *core.AppServer, db *gorm.DB, service *suno.Service, uploader *oss.UploaderManager, userService *service.UserService, moderationManager *moderation.ServiceManager) *SunoHandler { return &SunoHandler{ BaseHandler: BaseHandler{ App: app, DB: db, }, - sunoService: service, - uploader: uploader, - userService: userService, + sunoService: service, + uploader: uploader, + userService: userService, + moderationManager: moderationManager, } } @@ -84,6 +87,29 @@ func (h *SunoHandler) Create(c *gin.Context) { return } + if h.App.SysConfig.Moderation.Enable { + moderationResult, err := h.moderationManager.GetService().Moderate(data.Prompt) + if err != nil { + logger.Error("failed to moderate content: ", err) + } + if moderationResult.Flagged { + // 记录违规内容 + moderation := model.Moderation{ + UserId: h.GetLoginUserId(c), + Source: types.ModerationSourceSuno, + Input: data.Prompt, + Result: utils.JsonEncode(moderationResult), + } + err = h.DB.Create(&moderation).Error + if err != nil { + logger.Error("failed to save moderation: ", err) + } + resp.ERROR(c, "当前创作内容包含敏感词，请重新输入！") + return + } + + } + user, err := h.GetLoginUser(c) if err != nil { resp.NotAuth(c) diff --git a/api/handler/video_handler.go b/api/handler/video_handler.go index 1ebba5e6..d55873d8 100644 --- a/api/handler/video_handler.go +++ b/api/handler/video_handler.go @@ -13,6 +13,7 @@ import ( "geekai/core/middleware" "geekai/core/types" "geekai/service" + "geekai/service/moderation" "geekai/service/oss" "geekai/service/video" "geekai/store/model" @@ -27,20 +28,22 @@ import ( type VideoHandler struct { BaseHandler - videoService *video.Service - uploader *oss.UploaderManager - userService *service.UserService + videoService *video.Service + uploader *oss.UploaderManager + userService *service.UserService + moderationManager *moderation.ServiceManager } -func NewVideoHandler(app *core.AppServer, db *gorm.DB, service *video.Service, uploader *oss.UploaderManager, userService *service.UserService) *VideoHandler { +func NewVideoHandler(app *core.AppServer, db *gorm.DB, service *video.Service, uploader *oss.UploaderManager, userService *service.UserService, moderationManager *moderation.ServiceManager) *VideoHandler { return &VideoHandler{ BaseHandler: BaseHandler{ App: app, DB: db, }, - videoService: service, - uploader: uploader, - userService: userService, + videoService: service, + uploader: uploader, + userService: userService, + moderationManager: moderationManager, } } @@ -78,6 +81,29 @@ func (h *VideoHandler) LumaCreate(c *gin.Context) { return } + if h.App.SysConfig.Moderation.Enable { + moderationResult, err := h.moderationManager.GetService().Moderate(data.Prompt) + if err != nil { + logger.Error("failed to moderate content: ", err) + } + if moderationResult.Flagged { + // 记录违规内容 + moderation := model.Moderation{ + UserId: h.GetLoginUserId(c), + Source: types.ModerationSourceVideo, + Input: data.Prompt, + Result: utils.JsonEncode(moderationResult), + } + err = h.DB.Create(&moderation).Error + if err != nil { + logger.Error("failed to save moderation: ", err) + } + resp.ERROR(c, "当前创作内容包含敏感词，请重新输入！") + return + } + + } + user, err := h.GetLoginUser(c) if err != nil { resp.NotAuth(c) diff --git a/api/main.go b/api/main.go index 7c308f76..33466274 100644 --- a/api/main.go +++ b/api/main.go @@ -247,6 +247,10 @@ func main() { fx.Provide(moderation.NewBaiduAIModeration), fx.Provide(moderation.NewTencentAIModeration), fx.Provide(moderation.NewServiceManager), + fx.Provide(admin.NewModerationHandler), + fx.Invoke(func(s *core.AppServer, h *admin.ModerationHandler) { + h.RegisterRoutes() + }), // 注册路由 fx.Invoke(func(s *core.AppServer, h *handler.ChatAppHandler) { diff --git a/api/store/model/moderation.go b/api/store/model/moderation.go index 8b261798..039dc0f5 100644 --- a/api/store/model/moderation.go +++ b/api/store/model/moderation.go @@ -5,7 +5,8 @@ import "time" type Moderation struct { Id uint `gorm:"column:id;primaryKey;autoIncrement" json:"id"` UserId uint `gorm:"column:user_id;type:int(11);not null;comment:用户ID" json:"user_id"` - Input string `gorm:"column:prompt;type:text;not null;comment:用户输入" json:"input"` + Source string `gorm:"column:source;type:varchar(255);not null;comment:敏感词来源" json:"source"` + Input string `gorm:"column:input;type:text;not null;comment:用户输入" json:"input"` Output string `gorm:"column:output;type:text;not null;comment:AI 输出" json:"output"` Result string `gorm:"column:result;type:text;not null;comment:鉴别结果" json:"result"` CreatedAt time.Time `gorm:"column:created_at;type:datetime;not null" json:"created_at"` diff --git a/web/src/assets/css/sd-task-dialog.scss b/web/src/assets/css/sd-task-dialog.scss index efe9314c..1cf6c951 100644 --- a/web/src/assets/css/sd-task-dialog.scss +++ b/web/src/assets/css/sd-task-dialog.scss @@ -4,7 +4,7 @@ .el-dialog__header { .el-dialog__title { - color: #f5f5f5; + color: var(--text-color-primary); } } @@ -91,4 +91,4 @@ // end el-row } } -} \ No newline at end of file +} diff --git a/web/src/components/admin/AdminSidebar.vue b/web/src/components/admin/AdminSidebar.vue index 86234672..172327f9 100644 --- a/web/src/components/admin/AdminSidebar.vue +++ b/web/src/components/admin/AdminSidebar.vue @@ -190,6 +190,11 @@ const items = [ index: '/admin/config/moderation', title: '审查配置', }, + { + icon: 'list', + index: '/admin/moderation/list', + title: '审核记录', + }, ], }, { diff --git a/web/src/router.js b/web/src/router.js index 4dd525c3..9d3148c3 100644 --- a/web/src/router.js +++ b/web/src/router.js @@ -213,7 +213,13 @@ const routes = [ path: '/admin/config/moderation', name: 'admin-config-moderation', meta: { title: '文本审查配置' }, - component: () => import('@/views/admin/settings/ModerationConfig.vue'), + component: () => import('@/views/admin/moderation/ModerationConfig.vue'), + }, + { + path: '/admin/moderation/list', + name: 'admin-moderation-list', + meta: { title: '文本审核记录' }, + component: () => import('@/views/admin/moderation/ModerationList.vue'), }, { path: '/admin/config/markmap', diff --git a/web/src/views/admin/settings/ModerationConfig.vue b/web/src/views/admin/moderation/ModerationConfig.vue similarity index 100% rename from web/src/views/admin/settings/ModerationConfig.vue rename to web/src/views/admin/moderation/ModerationConfig.vue diff --git a/web/src/views/admin/moderation/ModerationList.vue b/web/src/views/admin/moderation/ModerationList.vue new file mode 100644 index 00000000..f79c0209 --- /dev/null +++ b/web/src/views/admin/moderation/ModerationList.vue @@ -0,0 +1,499 @@ + + + + +