文本审核记录功能完成

api/handler/admin/moderation_handler.go

@@ -0,0 +1,231 @@
+package admin
+
+// * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+// * Copyright 2023 The Geek-AI Authors. All rights reserved.
+// * Use of this source code is governed by a Apache-2.0 license
+// * that can be found in the LICENSE file.
+// * @Author yangjian102621@163.com
+// * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+import (
+	"geekai/core"
+	"geekai/core/middleware"
+	"geekai/core/types"
+	"geekai/handler"
+	"geekai/store/model"
+	"geekai/utils"
+	"geekai/utils/resp"
+
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+)
+
+type ModerationHandler struct {
+	handler.BaseHandler
+}
+
+func NewModerationHandler(app *core.AppServer, db *gorm.DB) *ModerationHandler {
+	return &ModerationHandler{BaseHandler: handler.BaseHandler{DB: db, App: app}}
+}
+
+// RegisterRoutes 注册路由
+func (h *ModerationHandler) RegisterRoutes() {
+	group := h.App.Engine.Group("/api/admin/moderation/")
+
+	// 需要管理员授权的接口
+	group.Use(middleware.AdminAuthMiddleware(h.App.Config.AdminSession.SecretKey, h.App.Redis))
+	{
+		group.POST("list", h.List)
+		group.GET("remove", h.Remove)
+		group.POST("batch-remove", h.BatchRemove)
+		group.GET("source-list", h.GetSourceList)
+	}
+}
+
+// List 获取文本审核记录列表
+func (h *ModerationHandler) List(c *gin.Context) {
+	var data struct {
+		Username  string `json:"username"`
+		Source    string `json:"source"`
+		StartDate string `json:"start_date"`
+		EndDate   string `json:"end_date"`
+		Page      int    `json:"page"`
+		PageSize  int    `json:"page_size"`
+	}
+
+	if err := c.ShouldBindJSON(&data); err != nil {
+		resp.ERROR(c, types.InvalidArgs)
+		return
+	}
+
+	session := h.DB.Session(&gorm.Session{})
+
+	// 构建查询条件
+	if data.Username != "" {
+		// 通过用户名查找用户ID
+		var user model.User
+		if err := h.DB.Where("username LIKE ?", "%"+data.Username+"%").First(&user).Error; err == nil {
+			session = session.Where("user_id", user.Id)
+		}
+	}
+
+	if data.Source != "" {
+		session = session.Where("source", data.Source)
+	}
+
+	if data.StartDate != "" && data.EndDate != "" {
+		startTime := data.StartDate + " 00:00:00"
+		endTime := data.EndDate + " 23:59:59"
+		session = session.Where("created_at >= ? AND created_at <= ?", startTime, endTime)
+	}
+
+	// 统计总数
+	var total int64
+	session.Model(&model.Moderation{}).Count(&total)
+
+	// 分页
+	page := data.Page
+	pageSize := data.PageSize
+	if page <= 0 {
+		page = 1
+	}
+	if pageSize <= 0 {
+		pageSize = 20
+	}
+
+	offset := (page - 1) * pageSize
+	session = session.Offset(offset).Limit(pageSize)
+
+	// 查询数据
+	var items []model.Moderation
+	err := session.Order("id DESC").Find(&items).Error
+	if err != nil {
+		resp.ERROR(c, err.Error())
+		return
+	}
+
+	// 获取用户信息
+	userIds := make([]uint, 0)
+	for _, item := range items {
+		userIds = append(userIds, item.UserId)
+	}
+
+	var users []model.User
+	if len(userIds) > 0 {
+		h.DB.Where("id IN ?", userIds).Find(&users)
+	}
+
+	userMap := make(map[uint]string)
+	for _, user := range users {
+		userMap[user.Id] = user.Username
+	}
+
+	// 转换为响应数据
+	list := make([]map[string]any, 0)
+	for _, item := range items {
+		var moderation types.ModerationResult
+		err := utils.JsonDecode(item.Result, &moderation)
+		if err != nil {
+			continue
+		}
+		var result []string
+		for value, label := range types.ModerationCategories {
+			if moderation.Categories[value] {
+				result = append(result, label)
+			}
+		}
+		list = append(list, map[string]any{
+			"id":         item.Id,
+			"user_id":    item.UserId,
+			"username":   userMap[item.UserId],
+			"source":     item.Source,
+			"input":      item.Input,
+			"output":     item.Output,
+			"result":     result,
+			"created_at": item.CreatedAt.Unix(),
+		})
+	}
+
+	resp.SUCCESS(c, map[string]any{
+		"items":     list,
+		"total":     total,
+		"page":      page,
+		"page_size": pageSize,
+	})
+}
+
+func (h *ModerationHandler) Remove(c *gin.Context) {
+	id := h.GetInt(c, "id", 0)
+	if id <= 0 {
+		resp.ERROR(c, types.InvalidArgs)
+		return
+	}
+
+	err := h.DB.Where("id", id).Delete(&model.Moderation{}).Error
+	if err != nil {
+		resp.ERROR(c, err.Error())
+		return
+	}
+	resp.SUCCESS(c)
+}
+
+// BatchRemove 批量删除文本审核记录
+func (h *ModerationHandler) BatchRemove(c *gin.Context) {
+	var data struct {
+		Ids []uint `json:"ids"`
+	}
+
+	if err := c.ShouldBindJSON(&data); err != nil {
+		resp.ERROR(c, types.InvalidArgs)
+		return
+	}
+
+	if len(data.Ids) == 0 {
+		resp.ERROR(c, "请选择要删除的记录")
+		return
+	}
+
+	err := h.DB.Where("id IN ?", data.Ids).Delete(&model.Moderation{}).Error
+	if err != nil {
+		resp.ERROR(c, err.Error())
+		return
+	}
+
+	resp.SUCCESS(c)
+}
+
+// 获取 source 列表
+func (h *ModerationHandler) GetSourceList(c *gin.Context) {
+	sources := []gin.H{
+		{
+			"id":   types.ModerationSourceChat,
+			"name": "AI对话",
+		},
+		{
+			"id":   types.ModerationSourceMJ,
+			"name": "Midjourney 绘图",
+		},
+		{
+			"id":   types.ModerationSourceDalle,
+			"name": "Dalle 绘图",
+		},
+		{
+			"id":   types.ModerationSourceSD,
+			"name": "StableDiffusion 绘图",
+		},
+		{
+			"id":   types.ModerationSourceSuno,
+			"name": "Suno 音乐",
+		},
+		{
+			"id":   types.ModerationSourceVideo,
+			"name": "视频生成",
+		},
+		{
+			"id":   types.ModerationSourceJiMeng,
+			"name": "即梦AI",
+		},
+	}
+
+	resp.SUCCESS(c, sources)
+}

api/handler/chat_handler.go

@@ -17,6 +17,7 @@ import (
 	"geekai/core/middleware"
 	"geekai/core/types"
 	"geekai/service"
+	"geekai/service/moderation"
 	"geekai/service/oss"
 	"geekai/store/model"
 	"geekai/store/vo"
@@ -62,21 +63,23 @@ type ChatInput struct {
 
 type ChatHandler struct {
 	BaseHandler
-	redis          *redis.Client
-	uploadManager  *oss.UploaderManager
-	licenseService *service.LicenseService
-	ReqCancelFunc  *types.LMap[string, context.CancelFunc] // HttpClient 请求取消 handle function
-	userService    *service.UserService
+	redis             *redis.Client
+	uploadManager     *oss.UploaderManager
+	licenseService    *service.LicenseService
+	ReqCancelFunc     *types.LMap[string, context.CancelFunc] // HttpClient 请求取消 handle function
+	userService       *service.UserService
+	moderationManager *moderation.ServiceManager
 }
 
-func NewChatHandler(app *core.AppServer, db *gorm.DB, redis *redis.Client, manager *oss.UploaderManager, licenseService *service.LicenseService, userService *service.UserService) *ChatHandler {
+func NewChatHandler(app *core.AppServer, db *gorm.DB, redis *redis.Client, manager *oss.UploaderManager, licenseService *service.LicenseService, userService *service.UserService, moderationManager *moderation.ServiceManager) *ChatHandler {
 	return &ChatHandler{
-		BaseHandler:    BaseHandler{App: app, DB: db},
-		redis:          redis,
-		uploadManager:  manager,
-		licenseService: licenseService,
-		ReqCancelFunc:  types.NewLMap[string, context.CancelFunc](),
-		userService:    userService,
+		BaseHandler:       BaseHandler{App: app, DB: db},
+		redis:             redis,
+		uploadManager:     manager,
+		licenseService:    licenseService,
+		ReqCancelFunc:     types.NewLMap[string, context.CancelFunc](),
+		userService:       userService,
+		moderationManager: moderationManager,
 	}
 }
 
@@ -309,6 +312,14 @@ func (h *ChatHandler) sendMessage(ctx context.Context, input ChatInput, c *gin.C
 	}
 	reqMgs := make([]any, 0)
 
+	// 添加引导提示词,防止模型生成违规内容
+	if h.App.SysConfig.Moderation.EnableGuide {
+		reqMgs = append(reqMgs, map[string]any{
+			"role":    "system",
+			"content": h.App.SysConfig.Moderation.GuidePrompt,
+		})
+	}
+
 	for i := len(chatCtx) - 1; i >= 0; i-- {
 		reqMgs = append(reqMgs, chatCtx[i])
 	}
@@ -352,16 +363,16 @@ func (h *ChatHandler) sendMessage(ctx context.Context, input ChatInput, c *gin.C
 	}
 
 	if len(imgList) > 0 {
-		imgList = append(imgList, map[string]interface{}{
+		imgList = append(imgList, map[string]any{
 			"type": "text",
 			"text": input.Prompt,
 		})
-		req.Messages = append(reqMgs, map[string]interface{}{
+		req.Messages = append(reqMgs, map[string]any{
 			"role":    "user",
 			"content": imgList,
 		})
 	} else {
-		req.Messages = append(reqMgs, map[string]interface{}{
+		req.Messages = append(reqMgs, map[string]any{
 			"role":    "user",
 			"content": finalPrompt,
 		})
@@ -557,6 +568,34 @@ func (h *ChatHandler) saveChatHistory(
 	promptCreatedAt time.Time,
 	replyCreatedAt time.Time) {
 
+	// 文本审核
+	if h.App.SysConfig.Moderation.Enable {
+		moderationResult, err := h.moderationManager.GetService().Moderate(usage.Content)
+		if err != nil {
+			logger.Error("failed to moderate content: ", err)
+		}
+		logger.Debugf("moderationResult: %+v", moderationResult)
+		if moderationResult.Flagged {
+			// 记录违规内容
+			moderation := model.Moderation{
+				UserId: userVo.Id,
+				Source: types.ModerationSourceChat,
+				Input:  usage.Prompt,
+				Output: usage.Content,
+				Result: utils.JsonEncode(moderationResult),
+			}
+			err = h.DB.Create(&moderation).Error
+			if err != nil {
+				logger.Error("failed to save moderation: ", err)
+			}
+			pushMessage(c, ChatEventError, "很抱歉，内容触发敏感词预警，AI 无法回答！！！")
+			// 更新用户算力
+			if input.ChatModel.Power > 0 {
+				h.subUserPower(userVo, input, 0, 0)
+			}
+			return
+		}
+	}
 	// 追加聊天记录
 	// for prompt
 	var promptTokens, replyTokens, totalTokens int

api/handler/dalle_handler.go

@@ -14,6 +14,7 @@ import (
 	"geekai/core/types"
 	"geekai/service"
 	"geekai/service/dalle"
+	"geekai/service/moderation"
 	"geekai/service/oss"
 	"geekai/store/model"
 	"geekai/store/vo"
@@ -26,16 +27,18 @@ import (
 
 type DallJobHandler struct {
 	BaseHandler
-	dallService *dalle.Service
-	uploader    *oss.UploaderManager
-	userService *service.UserService
+	dallService       *dalle.Service
+	uploader          *oss.UploaderManager
+	userService       *service.UserService
+	moderationManager *moderation.ServiceManager
 }
 
-func NewDallJobHandler(app *core.AppServer, db *gorm.DB, service *dalle.Service, manager *oss.UploaderManager, userService *service.UserService) *DallJobHandler {
+func NewDallJobHandler(app *core.AppServer, db *gorm.DB, service *dalle.Service, manager *oss.UploaderManager, userService *service.UserService, moderationManager *moderation.ServiceManager) *DallJobHandler {
 	return &DallJobHandler{
-		dallService: service,
-		uploader:    manager,
-		userService: userService,
+		dallService:       service,
+		uploader:          manager,
+		userService:       userService,
+		moderationManager: moderationManager,
 		BaseHandler: BaseHandler{
 			App: app,
 			DB:  db,
@@ -69,6 +72,29 @@ func (h *DallJobHandler) Image(c *gin.Context) {
 		return
 	}
 
+	// 文本审核
+	if h.App.SysConfig.Moderation.Enable {
+		moderationResult, err := h.moderationManager.GetService().Moderate(data.Prompt)
+		if err != nil {
+			logger.Error("failed to moderate content: ", err)
+		}
+		if moderationResult.Flagged {
+			// 记录违规内容
+			moderation := model.Moderation{
+				UserId: h.GetLoginUserId(c),
+				Source: types.ModerationSourceDalle,
+				Input:  data.Prompt,
+				Result: utils.JsonEncode(moderationResult),
+			}
+			err = h.DB.Create(&moderation).Error
+			if err != nil {
+				logger.Error("failed to save moderation: ", err)
+			}
+			resp.ERROR(c, "当前创作内容包含敏感词，提示词未通过文本审核，请重新输入！")
+			return
+		}
+	}
+
 	var chatModel model.ChatModel
 	if res := h.DB.Where("id = ?", data.ModelId).First(&chatModel); res.Error != nil {
 		resp.ERROR(c, "模型不存在")

api/handler/jimeng_handler.go

@@ -7,6 +7,7 @@ import (
 	"geekai/core/types"
 	"geekai/service"
 	"geekai/service/jimeng"
+	"geekai/service/moderation"
 	"geekai/store/model"
 	"geekai/store/vo"
 	"geekai/utils"
@@ -19,16 +20,18 @@ import (
 // JimengHandler 即梦AI处理器
 type JimengHandler struct {
 	BaseHandler
-	jimengService *jimeng.Service
-	userService   *service.UserService
+	jimengService     *jimeng.Service
+	userService       *service.UserService
+	moderationManager *moderation.ServiceManager
 }
 
 // NewJimengHandler 创建即梦AI处理器
-func NewJimengHandler(app *core.AppServer, jimengService *jimeng.Service, db *gorm.DB, userService *service.UserService) *JimengHandler {
+func NewJimengHandler(app *core.AppServer, jimengService *jimeng.Service, db *gorm.DB, userService *service.UserService, moderationManager *moderation.ServiceManager) *JimengHandler {
 	return &JimengHandler{
-		BaseHandler:   BaseHandler{App: app, DB: db},
-		jimengService: jimengService,
-		userService:   userService,
+		BaseHandler:       BaseHandler{App: app, DB: db},
+		jimengService:     jimengService,
+		userService:       userService,
+		moderationManager: moderationManager,
 	}
 }
 
@@ -75,6 +78,31 @@ func (h *JimengHandler) CreateTask(c *gin.Context) {
 		resp.ERROR(c, types.InvalidArgs)
 		return
 	}
+
+	// 文本审核
+	if h.App.SysConfig.Moderation.Enable {
+		moderationResult, err := h.moderationManager.GetService().Moderate(req.Prompt)
+		if err != nil {
+			logger.Error("failed to moderate content: ", err)
+		}
+		if moderationResult.Flagged {
+			// 记录违规内容
+			moderation := model.Moderation{
+				UserId: h.GetLoginUserId(c),
+				Source: types.ModerationSourceJiMeng,
+				Input:  req.Prompt,
+				Result: utils.JsonEncode(moderationResult),
+			}
+			err = h.DB.Create(&moderation).Error
+			if err != nil {
+				logger.Error("failed to save moderation: ", err)
+			}
+			resp.ERROR(c, "当前创作内容包含敏感词，请重新输入！")
+			return
+		}
+
+	}
+
 	// 新增：除图像特效外，其他任务类型必须有提示词
 	if req.TaskType != "image_effects" && req.Prompt == "" {
 		resp.ERROR(c, "提示词不能为空")

api/handler/mj_handler.go

@@ -14,6 +14,7 @@ import (
 	"geekai/core/types"
 	"geekai/service"
 	"geekai/service/mj"
+	"geekai/service/moderation"
 	"geekai/service/oss"
 	"geekai/store/model"
 	"geekai/store/vo"
@@ -28,18 +29,20 @@ import (
 
 type MidJourneyHandler struct {
 	BaseHandler
-	mjService   *mj.Service
-	snowflake   *service.Snowflake
-	uploader    *oss.UploaderManager
-	userService *service.UserService
+	mjService         *mj.Service
+	snowflake         *service.Snowflake
+	uploader          *oss.UploaderManager
+	userService       *service.UserService
+	moderationManager *moderation.ServiceManager
 }
 
-func NewMidJourneyHandler(app *core.AppServer, db *gorm.DB, snowflake *service.Snowflake, service *mj.Service, manager *oss.UploaderManager, userService *service.UserService) *MidJourneyHandler {
+func NewMidJourneyHandler(app *core.AppServer, db *gorm.DB, snowflake *service.Snowflake, service *mj.Service, manager *oss.UploaderManager, userService *service.UserService, moderationManager *moderation.ServiceManager) *MidJourneyHandler {
 	return &MidJourneyHandler{
-		snowflake:   snowflake,
-		mjService:   service,
-		uploader:    manager,
-		userService: userService,
+		snowflake:         snowflake,
+		mjService:         service,
+		uploader:          manager,
+		userService:       userService,
+		moderationManager: moderationManager,
 		BaseHandler: BaseHandler{
 			App: app,
 			DB:  db,
@@ -110,6 +113,29 @@ func (h *MidJourneyHandler) Image(c *gin.Context) {
 		return
 	}
 
+	// 文本审核
+	if h.App.SysConfig.Moderation.Enable {
+		moderationResult, err := h.moderationManager.GetService().Moderate(data.Prompt)
+		if err != nil {
+			logger.Error("failed to moderate content: ", err)
+		}
+		if moderationResult.Flagged {
+			// 记录违规内容
+			moderation := model.Moderation{
+				UserId: h.GetLoginUserId(c),
+				Source: types.ModerationSourceMJ,
+				Input:  data.Prompt,
+				Result: utils.JsonEncode(moderationResult),
+			}
+			err = h.DB.Create(&moderation).Error
+			if err != nil {
+				logger.Error("failed to save moderation: ", err)
+			}
+			resp.ERROR(c, "当前创作内容包含敏感词，请重新输入！")
+			return
+		}
+	}
+
 	var params = ""
 	if data.Rate != "" && !strings.Contains(params, "--ar") {
 		params += " --ar " + data.Rate

api/handler/sd_handler.go

@@ -13,6 +13,7 @@ import (
 	"geekai/core/middleware"
 	"geekai/core/types"
 	"geekai/service"
+	"geekai/service/moderation"
 	"geekai/service/oss"
 	"geekai/service/sd"
 	"geekai/store"
@@ -29,12 +30,13 @@ import (
 
 type SdJobHandler struct {
 	BaseHandler
-	redis       *redis.Client
-	sdService   *sd.Service
-	uploader    *oss.UploaderManager
-	snowflake   *service.Snowflake
-	leveldb     *store.LevelDB
-	userService *service.UserService
+	redis             *redis.Client
+	sdService         *sd.Service
+	uploader          *oss.UploaderManager
+	snowflake         *service.Snowflake
+	leveldb           *store.LevelDB
+	userService       *service.UserService
+	moderationManager *moderation.ServiceManager
 }
 
 func NewSdJobHandler(app *core.AppServer,
@@ -43,13 +45,15 @@ func NewSdJobHandler(app *core.AppServer,
 	manager *oss.UploaderManager,
 	snowflake *service.Snowflake,
 	userService *service.UserService,
-	levelDB *store.LevelDB) *SdJobHandler {
+	levelDB *store.LevelDB,
+	moderationManager *moderation.ServiceManager) *SdJobHandler {
 	return &SdJobHandler{
-		sdService:   service,
-		uploader:    manager,
-		snowflake:   snowflake,
-		leveldb:     levelDB,
-		userService: userService,
+		sdService:         service,
+		uploader:          manager,
+		snowflake:         snowflake,
+		leveldb:           levelDB,
+		userService:       userService,
+		moderationManager: moderationManager,
 		BaseHandler: BaseHandler{
 			App: app,
 			DB:  db,
@@ -102,6 +106,29 @@ func (h *SdJobHandler) Image(c *gin.Context) {
 		return
 	}
 
+	if h.App.SysConfig.Moderation.Enable {
+		moderationResult, err := h.moderationManager.GetService().Moderate(data.Prompt)
+		if err != nil {
+			logger.Error("failed to moderate content: ", err)
+		}
+		if moderationResult.Flagged {
+			// 记录违规内容
+			moderation := model.Moderation{
+				UserId: h.GetLoginUserId(c),
+				Source: types.ModerationSourceSD,
+				Input:  data.Prompt,
+				Result: utils.JsonEncode(moderationResult),
+			}
+			err = h.DB.Create(&moderation).Error
+			if err != nil {
+				logger.Error("failed to save moderation: ", err)
+			}
+			resp.ERROR(c, "当前创作内容包含敏感词，请重新输入！")
+			return
+		}
+
+	}
+
 	if data.Width <= 0 {
 		data.Width = 512
 	}

api/handler/suno_handler.go

@@ -13,6 +13,7 @@ import (
 	"geekai/core/middleware"
 	"geekai/core/types"
 	"geekai/service"
+	"geekai/service/moderation"
 	"geekai/service/oss"
 	"geekai/service/suno"
 	"geekai/store/model"
@@ -27,20 +28,22 @@ import (
 
 type SunoHandler struct {
 	BaseHandler
-	sunoService *suno.Service
-	uploader    *oss.UploaderManager
-	userService *service.UserService
+	sunoService       *suno.Service
+	uploader          *oss.UploaderManager
+	userService       *service.UserService
+	moderationManager *moderation.ServiceManager
 }
 
-func NewSunoHandler(app *core.AppServer, db *gorm.DB, service *suno.Service, uploader *oss.UploaderManager, userService *service.UserService) *SunoHandler {
+func NewSunoHandler(app *core.AppServer, db *gorm.DB, service *suno.Service, uploader *oss.UploaderManager, userService *service.UserService, moderationManager *moderation.ServiceManager) *SunoHandler {
 	return &SunoHandler{
 		BaseHandler: BaseHandler{
 			App: app,
 			DB:  db,
 		},
-		sunoService: service,
-		uploader:    uploader,
-		userService: userService,
+		sunoService:       service,
+		uploader:          uploader,
+		userService:       userService,
+		moderationManager: moderationManager,
 	}
 }
 
@@ -84,6 +87,29 @@ func (h *SunoHandler) Create(c *gin.Context) {
 		return
 	}
 
+	if h.App.SysConfig.Moderation.Enable {
+		moderationResult, err := h.moderationManager.GetService().Moderate(data.Prompt)
+		if err != nil {
+			logger.Error("failed to moderate content: ", err)
+		}
+		if moderationResult.Flagged {
+			// 记录违规内容
+			moderation := model.Moderation{
+				UserId: h.GetLoginUserId(c),
+				Source: types.ModerationSourceSuno,
+				Input:  data.Prompt,
+				Result: utils.JsonEncode(moderationResult),
+			}
+			err = h.DB.Create(&moderation).Error
+			if err != nil {
+				logger.Error("failed to save moderation: ", err)
+			}
+			resp.ERROR(c, "当前创作内容包含敏感词，请重新输入！")
+			return
+		}
+
+	}
+
 	user, err := h.GetLoginUser(c)
 	if err != nil {
 		resp.NotAuth(c)

api/handler/video_handler.go

@@ -13,6 +13,7 @@ import (
 	"geekai/core/middleware"
 	"geekai/core/types"
 	"geekai/service"
+	"geekai/service/moderation"
 	"geekai/service/oss"
 	"geekai/service/video"
 	"geekai/store/model"
@@ -27,20 +28,22 @@ import (
 
 type VideoHandler struct {
 	BaseHandler
-	videoService *video.Service
-	uploader     *oss.UploaderManager
-	userService  *service.UserService
+	videoService      *video.Service
+	uploader          *oss.UploaderManager
+	userService       *service.UserService
+	moderationManager *moderation.ServiceManager
 }
 
-func NewVideoHandler(app *core.AppServer, db *gorm.DB, service *video.Service, uploader *oss.UploaderManager, userService *service.UserService) *VideoHandler {
+func NewVideoHandler(app *core.AppServer, db *gorm.DB, service *video.Service, uploader *oss.UploaderManager, userService *service.UserService, moderationManager *moderation.ServiceManager) *VideoHandler {
 	return &VideoHandler{
 		BaseHandler: BaseHandler{
 			App: app,
 			DB:  db,
 		},
-		videoService: service,
-		uploader:     uploader,
-		userService:  userService,
+		videoService:      service,
+		uploader:          uploader,
+		userService:       userService,
+		moderationManager: moderationManager,
 	}
 }
 
@@ -78,6 +81,29 @@ func (h *VideoHandler) LumaCreate(c *gin.Context) {
 		return
 	}
 
+	if h.App.SysConfig.Moderation.Enable {
+		moderationResult, err := h.moderationManager.GetService().Moderate(data.Prompt)
+		if err != nil {
+			logger.Error("failed to moderate content: ", err)
+		}
+		if moderationResult.Flagged {
+			// 记录违规内容
+			moderation := model.Moderation{
+				UserId: h.GetLoginUserId(c),
+				Source: types.ModerationSourceVideo,
+				Input:  data.Prompt,
+				Result: utils.JsonEncode(moderationResult),
+			}
+			err = h.DB.Create(&moderation).Error
+			if err != nil {
+				logger.Error("failed to save moderation: ", err)
+			}
+			resp.ERROR(c, "当前创作内容包含敏感词，请重新输入！")
+			return
+		}
+
+	}
+
 	user, err := h.GetLoginUser(c)
 	if err != nil {
 		resp.NotAuth(c)