Jimeng VirtualHuman and actionTransfer is ready

2026-05-10 19:54:25 +08:00 · 2025-09-15 20:29:46 +08:00
parent 822d1831cd
commit c4b44d84e3
12 changed files with 1155 additions and 151 deletions
--- a/api/core/types/jimeng.go
+++ b/api/core/types/jimeng.go
@@ -46,14 +46,17 @@ const (
 type JimengTaskRequest struct {
 	TaskType JMTaskType `json:"type"`    // 任务类型
 	ReqKey   string     `json:"req_key"` // 请求Key
+	Action   string     `json:"action"`  // 请求Action
 	Power    int        `json:"power"`   // 消耗算力
 	// 公共参数
 	Prompt    string   `json:"prompt,omitempty"`
 	ImageUrls []string `json:"image_urls,omitempty"`

 	// 图片生成参数
-	Size      string `json:"size,omitempty"`
-	UsePreLLM bool   `json:"use_pre_llm,omitempty"`
+	Size        string  `json:"size,omitempty"`
+	UsePreLLM   bool    `json:"use_pre_llm,omitempty"`
+	Scale       float64 `json:"scale,omitempty"`
+	ForceSingle bool    `json:"force_single,omitempty"`

 	// 视频生成参数
 	Duration       int    `json:"duration,omitempty"`    // 视频时长，单位：秒
@@ -62,7 +65,8 @@ type JimengTaskRequest struct {
 	CameraStrength string `json:"camera_strength,omitempty"` // 运镜强度

 	// 数字人视频生成参数
-	AudioURL string `json:"audio_url,omitempty"` // 音频URL
+	AudioURL     string `json:"audio_url,omitempty"`     // 音频URL
+	RecognizeKey string `json:"recognize_key,omitempty"` // 识别主体请求Key

 	// 视频动作迁移参数
 	VideoURL string `json:"video_url,omitempty"` // 动作视频URL
--- a/api/handler/jimeng_handler.go
+++ b/api/handler/jimeng_handler.go
@@ -97,7 +97,7 @@ func (h *JimengHandler) CreateTask(c *gin.Context) {
 	// 获取算力消耗
 	powerCost, err := h.getTaskPower(req)
 	if err != nil {
-		resp.ERROR(c, "计算任务消耗积分失败")
+		resp.ERROR(c, "计算任务消耗积分失败: "+err.Error())
 		return
 	}

@@ -105,6 +105,7 @@ func (h *JimengHandler) CreateTask(c *gin.Context) {
 		resp.ERROR(c, fmt.Sprintf("算力不足，需要%d算力", powerCost))
 		return
 	}
+	req.Power = powerCost

 	job, err := h.jimengService.CreateTask(user.Id, &req)
 	if err != nil {
@@ -116,12 +117,27 @@ func (h *JimengHandler) CreateTask(c *gin.Context) {
 	h.userService.DecreasePower(user.Id, powerCost, model.PowerLog{
 		Type:   types.PowerConsume,
 		Model:  job.ReqKey,
-		Remark: fmt.Sprintf("%s，任务ID：%d", req.ReqKey, job.Id),
+		Remark: h.getTaskRemark(req, job.Id),
 	})

 	resp.SUCCESS(c)
 }

+func (h *JimengHandler) getTaskRemark(req types.JimengTaskRequest, jobId uint) string {
+	remark := fmt.Sprintf("即梦任务%s，任务ID：%d", req.ReqKey, jobId)
+	switch req.TaskType {
+	case types.JMTaskTypeImage:
+		remark = fmt.Sprintf("即梦图片生成，任务ID：%d，%d积分/张", jobId, h.App.SysConfig.Jimeng.Power.Image)
+	case types.JMTaskTypeVideo:
+		remark = fmt.Sprintf("即梦视频生成，任务ID：%d，%d积分/秒, %d秒", jobId, h.App.SysConfig.Jimeng.Power.Video, req.Power/h.App.SysConfig.Jimeng.Power.Video)
+	case types.JMTaskTypeVirtualHuman:
+		remark = fmt.Sprintf("即梦数字人视频生成，任务ID：%d，%d积分/秒, %d秒", jobId, h.App.SysConfig.Jimeng.Power.VirtualHuman, req.Power/h.App.SysConfig.Jimeng.Power.VirtualHuman)
+	case types.JMTaskTypeActionTransfer:
+		remark = fmt.Sprintf("即梦视频动作迁移，任务ID：%d，%d积分/秒, %d秒", jobId, h.App.SysConfig.Jimeng.Power.ActionTransfer, req.Power/h.App.SysConfig.Jimeng.Power.ActionTransfer)
+	}
+	return remark
+}
+
 // Jobs 获取任务列表
 func (h *JimengHandler) Jobs(c *gin.Context) {
 	userId := h.GetLoginUserId(c)
@@ -219,7 +235,8 @@ func (h *JimengHandler) Remove(c *gin.Context) {
 	}

 	// 失败任务删除后退回算力
-	if job.Status != types.JMTaskStatusFailed {
+	if job.Status == types.JMTaskStatusFailed {
+		logger.Infof("delete jimeng job failed, refund power: %d", job.Power)
 		err = h.userService.IncreasePower(user.Id, job.Power, model.PowerLog{
 			Type:   types.PowerRefund,
 			Model:  job.ReqKey,
@@ -284,6 +301,7 @@ func (h *JimengHandler) Retry(c *gin.Context) {

 // getPowerFromConfig 从配置中获取指定类型的算力消耗
 func (h *JimengHandler) getTaskPower(req types.JimengTaskRequest) (int, error) {
+	logger.Debugf("getTaskPower req: %+v", req)
 	config := h.App.SysConfig.Jimeng
 	switch req.TaskType {
 	case types.JMTaskTypeImage:
@@ -295,10 +313,24 @@ func (h *JimengHandler) getTaskPower(req types.JimengTaskRequest) (int, error) {
 		return config.Power.Video * req.Duration, nil
 	case types.JMTaskTypeVirtualHuman:
 		// TODO 计算音频时长
-		return config.Power.VirtualHuman, nil
+		if req.AudioURL == "" {
+			return 0, errors.New("音频URL不能为空")
+		}
+		audioDuration, err := utils.AudioDurationFromURL(req.AudioURL)
+		if err != nil {
+			return 0, err
+		}
+		return config.Power.VirtualHuman * int(audioDuration.Seconds()), nil
 	case types.JMTaskTypeActionTransfer:
 		// TODO 计算视频时长
-		return config.Power.ActionTransfer, nil
+		if req.VideoURL == "" {
+			return 0, errors.New("视频URL不能为空")
+		}
+		videoDuration, err := utils.VideoDurationMP4FromURL(req.VideoURL)
+		if err != nil {
+			return 0, err
+		}
+		return config.Power.ActionTransfer * int(videoDuration.Seconds()), nil
 	default:
 		return 0, errors.New("任务类型不支持")
 	}
--- a/api/service/jimeng/client.go
+++ b/api/service/jimeng/client.go
@@ -3,11 +3,13 @@ package jimeng
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"geekai/core/types"
 	"net/http"
 	"net/url"
 	"strings"
+	"time"

 	"github.com/volcengine/volc-sdk-golang/base"
 	"github.com/volcengine/volc-sdk-golang/service/visual"
@@ -54,6 +56,22 @@ func (c *Client) UpdateConfig(config types.JimengConfig) error {
 				"Version": []string{"2022-08-31"},
 			},
 		},
+		"CVSubmitTask": {
+			Method: http.MethodPost,
+			Path:   "/",
+			Query: url.Values{
+				"Action":  []string{"CVSubmitTask"},
+				"Version": []string{"2022-08-31"},
+			},
+		},
+		"CVGetResult": {
+			Method: http.MethodPost,
+			Path:   "/",
+			Query: url.Values{
+				"Action":  []string{"CVGetResult"},
+				"Version": []string{"2022-08-31"},
+			},
+		},
 		"CVProcess": {
 			Method: http.MethodPost,
 			Path:   "/",
@@ -75,6 +93,22 @@ func (c *Client) UpdateConfig(config types.JimengConfig) error {
 	return c.testConnection()
 }

+// GetErrorMessage 根据错误代码获取对应的错误信息
+func GetErrorMessage(code int) string {
+	if message, exists := errorCodeMessages[code]; exists {
+		return message
+	}
+	return fmt.Sprintf("未知错误代码: %d", code)
+}
+
+// HandleResponseError 处理响应错误，根据错误代码返回详细的错误信息
+func HandleResponseError(code int, message string) error {
+	if code == ECSuccess {
+		return nil
+	}
+	return errors.New(GetErrorMessage(code))
+}
+
 // testConnection 测试即梦AI连接
 func (c *Client) testConnection() error {

@@ -84,7 +118,7 @@ func (c *Client) testConnection() error {
 		TaskId: "test_task_id_12345",
 	}

-	_, err := c.QueryTask(testReq)
+	_, err := c.QueryTask(testReq, ASyncActionGetResult)
 	// 即使任务不存在，只要不是认证错误就说明连接正常
 	if err != nil {
 		// 检查是否是认证错误
@@ -105,17 +139,16 @@ func (c *Client) SubmitTask(req map[string]any) (*SubmitTaskResponse, error) {
 		return nil, fmt.Errorf("marshal request failed: %w", err)
 	}

-	// 单独处理图片特效任务
-	if req["req_key"] == ImageEffectReqKey {
-		req["image_input1"] = req["image_urls"].([]any)[0]
-		delete(req, "image_urls")
-	}
-
 	// 直接使用序列化后的字节
 	jsonBody := reqBodyBytes
+	action := ASyncActionSubmit
+	if v, ok := req["action"]; ok {
+		action = v.(string)
+		delete(req, "action")
+	}

 	// 调用SDK的JSON方法
-	respBody, statusCode, err := c.visual.Client.Json("CVSync2AsyncSubmitTask", nil, string(jsonBody))
+	respBody, statusCode, err := c.visual.Client.Json(action, nil, string(jsonBody))
 	if err != nil {
 		return nil, fmt.Errorf("submit task failed (status: %d): %w", statusCode, err)
 	}
@@ -128,11 +161,70 @@ func (c *Client) SubmitTask(req map[string]any) (*SubmitTaskResponse, error) {
 		return nil, fmt.Errorf("unmarshal response failed: %w", err)
 	}

+	// 检查响应错误代码
+	if err := HandleResponseError(result.Code, result.Message); err != nil {
+		return nil, err
+	}
+
 	return &result, nil
 }

+// 识别数字人主体
+func (c *Client) AvatarRecognition(imgUrl string, reqKey string) error {
+	params := map[string]any{
+		"image_url": imgUrl,
+		"req_key":   reqKey,
+	}
+	reqBodyBytes, err := json.Marshal(params)
+	if err != nil {
+		return fmt.Errorf("marshal request failed: %w", err)
+	}
+	// 调用SDK的JSON方法
+	respBody, statusCode, err := c.visual.Client.Json(SyncActionSubmit, nil, string(reqBodyBytes))
+	if err != nil {
+		return fmt.Errorf("submit task failed (status: %d): %w", statusCode, err)
+	}
+
+	// 解析响应
+	var result SubmitTaskResponse
+	if err := json.Unmarshal(respBody, &result); err != nil {
+		return fmt.Errorf("unmarshal response failed: %w", err)
+	}
+
+	// 检查响应错误代码
+	if err := HandleResponseError(result.Code, result.Message); err != nil {
+		return err
+	}
+
+	// 等待任务完成
+	for {
+		resp, err := c.QueryTask(&QueryTaskRequest{
+			ReqKey: reqKey,
+			TaskId: result.Data.TaskId,
+		}, SyncActionGetResult)
+		if err != nil {
+			return fmt.Errorf("query task failed: %w", err)
+		}
+		if resp.Data.Status != types.JMTaskStatusDone {
+			time.Sleep(time.Second * 3)
+			continue
+		}
+		var respData map[string]int
+		if err := json.Unmarshal([]byte(resp.Data.RespData), &respData); err != nil {
+			return fmt.Errorf("unmarshal response failed: %w", err)
+		}
+		logger.Debugf("Jimeng AvatarRecognition Response: %+v", resp)
+		if respData["status"] == 1 {
+			return nil
+		} else {
+			return errors.New("不包含人、类人、拟人等主体")
+		}
+
+	}
+}
+
 // QueryTask 查询任务结果
-func (c *Client) QueryTask(req *QueryTaskRequest) (*QueryTaskResponse, error) {
+func (c *Client) QueryTask(req *QueryTaskRequest, action string) (*QueryTaskResponse, error) {
 	// 序列化请求
 	jsonBody, err := json.Marshal(req)
 	if err != nil {
@@ -140,7 +232,7 @@ func (c *Client) QueryTask(req *QueryTaskRequest) (*QueryTaskResponse, error) {
 	}

 	// 调用SDK的JSON方法
-	respBody, statusCode, err := c.visual.Client.Json("CVSync2AsyncGetResult", nil, string(jsonBody))
+	respBody, statusCode, err := c.visual.Client.Json(action, nil, string(jsonBody))
 	if err != nil {
 		return nil, fmt.Errorf("query task failed (status: %d): %w", statusCode, err)
 	}
@@ -153,6 +245,11 @@ func (c *Client) QueryTask(req *QueryTaskRequest) (*QueryTaskResponse, error) {
 		return nil, fmt.Errorf("unmarshal response failed: %w", err)
 	}

+	// 检查响应错误代码
+	if err := HandleResponseError(result.Code, result.Message); err != nil {
+		return nil, err
+	}
+
 	return &result, nil
 }

--- a/api/service/jimeng/service.go
+++ b/api/service/jimeng/service.go
@@ -160,7 +160,12 @@ func (s *Service) ProcessTask(jobId uint) error {
 		return s.handleTaskError(job.Id, fmt.Sprintf("build task request failed: %v", err))
 	}

-	logger.Debugf("提交即梦任务: %+v", params)
+	// 数字人任务，先识别主体
+	if req.TaskType == types.JMTaskTypeVirtualHuman {
+		if err := s.client.AvatarRecognition(req.ImageUrls[0], req.RecognizeKey); err != nil {
+			return s.handleTaskError(job.Id, fmt.Sprintf("avatar recognition failed: %v", err))
+		}
+	}

 	// 同步任务 ，后台执行
 	if req.ReqKey == DoubaoSeedream40ReqKey {
@@ -195,6 +200,7 @@ func (s *Service) ProcessTask(jobId uint) error {
 		return nil
 	}

+	logger.Debugf("提交即梦任务: %+v", params)
 	// 异步任务 ，前台执行
 	resp, err := s.client.SubmitTask(params)
 	if err != nil {
@@ -245,6 +251,21 @@ func (s *Service) buildTaskRequest(req *types.JimengTaskRequest) (map[string]any
 		delete(params, "duration")
 	}

+	// 单独处理图片特效任务
+	if req.ReqKey == ImageEffectReqKey {
+		params["image_input1"] = req.ImageUrls[0]
+		delete(params, "image_urls")
+	}
+
+	// 动作迁移，数字人任务参数处理
+	if req.TaskType == types.JMTaskTypeVirtualHuman || req.TaskType == types.JMTaskTypeActionTransfer {
+		params["image_url"] = req.ImageUrls[0]
+		delete(params, "image_urls")
+	}
+	if req.RecognizeKey != "" {
+		delete(params, "recognize_key")
+	}
+
 	// 删除多余参数，剩下的就是各个任务自己专有参数了
 	delete(params, "type")
 	delete(params, "power")
@@ -280,7 +301,7 @@ func (s *Service) pollTaskStatus() {
 				ReqKey:  job.ReqKey,
 				TaskId:  job.TaskId,
 				ReqJson: `{"return_url":true}`,
-			})
+			}, ASyncActionGetResult)

 			if err != nil {
 				s.handleTaskError(job.Id, fmt.Sprintf("query task failed: %s", err.Error()))
--- a/api/service/jimeng/types.go
+++ b/api/service/jimeng/types.go
@@ -51,6 +51,68 @@ type QueryTaskResponse struct {

 const CodeSuccess = 10000

+// 即梦AI错误代码常量
+const (
+	// 成功
+	ECSuccess = 10000
+
+	// 请求参数错误 (50200-50215)
+	ECReqInvalidArgs     = 50200 // 参数错误
+	ECReqMissingArgs     = 50201 // 缺少参数
+	ECParseArgs          = 50204 // 参数类型错误/参数缺失
+	ECImageSizeLimited   = 50205 // 图像尺寸超过限制
+	ECImageEmpty         = 50206 // 请求参数中没有获取到图像
+	ECImageDecodeError   = 50207 // 图像解码错误
+	ECVideoEmpty         = 50209 // 请求参数中没有获取到视频
+	ECVideoDecodeError   = 50210 // 视频解码错误
+	ECVideoSizeLimited   = 50211 // 视频尺寸超过限制
+	ECReqBodySizeLimited = 50213 // 请求Body过大
+	ECVideoTimeTooLong   = 50214 // 输入视频时长过大
+	ECRPCProcess         = 50215 // 请求处理失败
+
+	// 算法服务错误 (60102-60208)
+	ECJPFaceDetect      = 60102 // 算法服务需要输入人脸图，但未检测到
+	ECFSLeaderRiskError = 60208 // 输入图片中包含敏感信息，未通过审核
+
+	// 权限和系统错误 (50400-50501)
+	ECAuth        = 50400 // 权限校验失败
+	ECReqMethod   = 50402 // 访问的接口不存在
+	ECReqLimit    = 50429 // 超过调用QPS限制
+	ECInternal    = 50500 // 服务器内部错误
+	ECRPCInternal = 50501 // 服务器内部RPC错误
+)
+
+// 错误代码到错误信息的映射
+var errorCodeMessages = map[int]string{
+	// 成功
+	ECSuccess: "请求成功",
+
+	// 请求参数错误
+	ECReqInvalidArgs:     "参数错误，检查入参及MIME类型",
+	ECReqMissingArgs:     "缺少参数，检查入参及MIME类型",
+	ECParseArgs:          "参数类型错误/参数缺失，检查入参及MIME类型",
+	ECImageSizeLimited:   "图像尺寸超过限制，参考接口文档入参要求部分",
+	ECImageEmpty:         "请求参数中没有获取到图像，检查入参",
+	ECImageDecodeError:   "图像解码错误：没有获取到图像或者通过image_base64参数传递图像是base64解码错误，检查输出图片或检查base64是否错误携带前缀",
+	ECVideoEmpty:         "请求参数中没有获取到视频。输入为视频时可能返回此错误，检查入参",
+	ECVideoDecodeError:   "视频解码错误。输入为视频时可能返回此错误，检查输入视频是否不正确",
+	ECVideoSizeLimited:   "视频尺寸超过限制。输入为视频时可能返回此错误，检查输入视频大小",
+	ECReqBodySizeLimited: "请求Body过大，超出接口限制，检查请求Body大小",
+	ECVideoTimeTooLong:   "输入视频时长过大，检查输入视频时长",
+	ECRPCProcess:         "由于输入的图片、视频、参数等不满足要求，导致请求处理失败。若接口文档中有具体说明，优先参考其具体含义，按照具体服务说明进行检查",
+
+	// 算法服务错误
+	ECJPFaceDetect:      "算法服务需要输入人脸图，但未检测到，检查输入图片是否包含人脸",
+	ECFSLeaderRiskError: "输入图片中包含敏感信息，未通过审核",
+
+	// 权限和系统错误
+	ECAuth:        "权限校验失败，请检查是否已创建应用并开通服务或签名，参考接入指南及快速接入",
+	ECReqMethod:   "访问的接口不存在，检查入参",
+	ECReqLimit:    "超过调用QPS限制，购买QPS增项包",
+	ECInternal:    "服务器内部错误，提工单",
+	ECRPCInternal: "服务器内部RPC错误，提工单",
+}
+
 // CreateTaskRequest 创建任务请求
 type CreateTaskRequest struct {
 	Type      types.JMTaskType `json:"type"`
@@ -65,3 +127,10 @@ const (
 	ImageEffectReqKey      = "i2i_multi_style_zx2x"
 	DoubaoSeedream40ReqKey = "doubao-seedream-4-0-250828"
 )
+
+const (
+	ASyncActionSubmit    = "CVSync2AsyncSubmitTask" // 异步提交任务
+	SyncActionSubmit     = "CVSubmitTask"           // 同步提交任务
+	ASyncActionGetResult = "CVSync2AsyncGetResult"  // 异步获取结果
+	SyncActionGetResult  = "CVGetResult"            // 同步获取结果
+)