mirror of
https://github.com/yangjian102621/geekai.git
synced 2026-08-29 02:17:13 +00:00
feat(release): migrate GeekAI v4.3.0 to open source
- Sync backend and frontend from GeekAI Plus v4.3.0 - Remove commercial License flows and update open-source deployment defaults - Preserve Docker Compose deployment and bump image tags to v4.3.0 BREAKING CHANGE: commercial License configuration and related endpoints are removed
This commit is contained in:
+420
-123
@@ -8,17 +8,15 @@ package service
|
||||
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"geekai/core/types"
|
||||
"geekai/store"
|
||||
"geekai/store/model"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/go-redis/redis/v8"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/schema"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -33,23 +31,21 @@ type MigrationService struct {
|
||||
db *gorm.DB
|
||||
redisClient *redis.Client
|
||||
appConfig *types.AppConfig
|
||||
levelDB *store.LevelDB
|
||||
}
|
||||
|
||||
func NewMigrationService(db *gorm.DB, redisClient *redis.Client, appConfig *types.AppConfig, levelDB *store.LevelDB) *MigrationService {
|
||||
func NewMigrationService(db *gorm.DB, redisClient *redis.Client, appConfig *types.AppConfig) *MigrationService {
|
||||
return &MigrationService{
|
||||
db: db,
|
||||
redisClient: redisClient,
|
||||
appConfig: appConfig,
|
||||
levelDB: levelDB,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MigrationService) StartMigrate() {
|
||||
// 表结构同步必须在对外服务前完成,避免缺列导致业务报错
|
||||
// 表结构迁移必须在业务服务启动前完成,避免新表和新列尚未创建就被后台任务查询。
|
||||
s.TableMigration()
|
||||
go func() {
|
||||
_ = s.MigrateConfig(s.appConfig)
|
||||
s.MigrateConfig(s.appConfig)
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -126,163 +122,386 @@ func (s *MigrationService) MigrateConfigContent() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 永不删除的保护列(大小写不敏感)
|
||||
var protectedColumns = map[string]struct{}{
|
||||
"id": {},
|
||||
"created_at": {},
|
||||
"updated_at": {},
|
||||
}
|
||||
|
||||
// allModels 全部需要同步的数据表 model
|
||||
func allModels() []any {
|
||||
return []any{
|
||||
// fullTableMigration 第一步:全量表迁移,同步所有表结构(新增表、新增字段、字段类型)
|
||||
// 适用于首次安装或导入旧版数据库后同步 schema,AutoMigrate 会补齐缺失的表和列
|
||||
func (s *MigrationService) fullTableMigration() {
|
||||
logger.Info("执行全量表迁移(同步 schema)...")
|
||||
models := []any{
|
||||
&model.Config{},
|
||||
&model.AdminUser{},
|
||||
&model.ChatApp{},
|
||||
&model.ApiKey{},
|
||||
&model.AppType{},
|
||||
&model.ChatApp{},
|
||||
&model.ChatModel{},
|
||||
&model.User{},
|
||||
&model.ChatItem{},
|
||||
&model.ChatMessage{},
|
||||
&model.ChatModel{},
|
||||
&model.Config{},
|
||||
&model.DallJob{},
|
||||
&model.File{},
|
||||
&model.Order{},
|
||||
&model.Product{},
|
||||
&model.Function{},
|
||||
&model.Menu{},
|
||||
&model.InviteCode{},
|
||||
&model.InviteLog{},
|
||||
&model.JimengJob{},
|
||||
&model.Menu{},
|
||||
&model.MidJourneyJob{},
|
||||
&model.Moderation{},
|
||||
&model.Order{},
|
||||
&model.PowerLog{},
|
||||
&model.Product{},
|
||||
&model.Redeem{},
|
||||
&model.SdJob{},
|
||||
&model.SunoJob{},
|
||||
&model.User{},
|
||||
&model.PowerLog{},
|
||||
&model.File{},
|
||||
&model.UserLoginLog{},
|
||||
&model.MidJourneyJob{},
|
||||
&model.SunoJob{},
|
||||
&model.VideoJob{},
|
||||
&model.JimengJob{},
|
||||
&model.PPTJob{},
|
||||
&model.Moderation{},
|
||||
&model.ImageJob{},
|
||||
}
|
||||
}
|
||||
|
||||
// 数据表迁移:先处理字段重命名(保数据),再全量同步 schema
|
||||
func (s *MigrationService) TableMigration() {
|
||||
logger.Info("开始数据表迁移...")
|
||||
s.renameColumns()
|
||||
if err := s.SyncAllModels(); err != nil {
|
||||
logger.Errorf("同步数据表字段失败: %v", err)
|
||||
if err := s.db.AutoMigrate(models...); err != nil {
|
||||
logger.Errorf("全量表迁移失败: %v", err)
|
||||
return
|
||||
}
|
||||
logger.Info("数据表迁移完成")
|
||||
logger.Info("全量表迁移完成")
|
||||
}
|
||||
|
||||
// renameColumns 只处理「改名」场景:删旧加新会丢数据,必须先 Rename
|
||||
func (s *MigrationService) renameColumns() {
|
||||
m := s.db.Migrator()
|
||||
// fixTableConstraints 第一步之后:根据模型定义修复各表的主键、自增和关键索引
|
||||
// 主要解决初始化 SQL 中缺少 AUTO_INCREMENT 或 PRIMARY KEY 导致插入失败的问题
|
||||
func (s *MigrationService) fixTableConstraints() {
|
||||
logger.Info("开始修复各表的主键、自增属性和索引...")
|
||||
|
||||
if m.HasColumn(&model.JimengJob{}, "task_params") {
|
||||
_ = m.RenameColumn(&model.JimengJob{}, "task_params", "params")
|
||||
// 当前数据库名
|
||||
var dbName string
|
||||
if err := s.db.Raw("SELECT DATABASE()").Scan(&dbName).Error; err != nil {
|
||||
logger.Errorf("获取当前数据库名失败: %v", err)
|
||||
return
|
||||
}
|
||||
if m.HasColumn(&model.Order{}, "pay_type") {
|
||||
_ = m.RenameColumn(&model.Order{}, "pay_type", "channel")
|
||||
if dbName == "" {
|
||||
logger.Warn("当前连接未选择数据库,跳过约束修复")
|
||||
return
|
||||
}
|
||||
if m.HasColumn(&model.Config{}, "config_json") {
|
||||
_ = m.RenameColumn(&model.Config{}, "config_json", "value")
|
||||
}
|
||||
if m.HasColumn(&model.Config{}, "marker") {
|
||||
_ = m.RenameColumn(&model.Config{}, "marker", "name")
|
||||
}
|
||||
if m.HasIndex(&model.Config{}, "idx_chatgpt_configs_key") {
|
||||
_ = m.DropIndex(&model.Config{}, "idx_chatgpt_configs_key")
|
||||
}
|
||||
if m.HasIndex(&model.Config{}, "marker") {
|
||||
_ = m.DropIndex(&model.Config{}, "marker")
|
||||
}
|
||||
}
|
||||
|
||||
// SyncAllModels 按 model 定义同步所有数据表:缺列新建,多余列删除
|
||||
func (s *MigrationService) SyncAllModels() error {
|
||||
var firstErr error
|
||||
for _, m := range allModels() {
|
||||
if err := s.syncModel(m); err != nil {
|
||||
logger.Errorf("同步 model %T 失败: %v", m, err)
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
// === 修复所有包含 id 字段的表的主键 + 自增 ===
|
||||
type columnInfo struct {
|
||||
TableName string `gorm:"column:TABLE_NAME"`
|
||||
ColumnName string `gorm:"column:COLUMN_NAME"`
|
||||
ColumnKey string `gorm:"column:COLUMN_KEY"`
|
||||
Extra string `gorm:"column:EXTRA"`
|
||||
DataType string `gorm:"column:DATA_TYPE"`
|
||||
}
|
||||
|
||||
var idColumns []columnInfo
|
||||
if err := s.db.Raw(`
|
||||
SELECT TABLE_NAME, COLUMN_NAME, COLUMN_KEY, EXTRA, DATA_TYPE
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = ? AND COLUMN_NAME = 'id'
|
||||
`, dbName).Scan(&idColumns).Error; err != nil {
|
||||
logger.Errorf("查询各表 id 字段信息失败: %v", err)
|
||||
} else {
|
||||
for _, col := range idColumns {
|
||||
if col.ColumnName == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// 检查当前表是否已经存在主键
|
||||
type pkInfo struct {
|
||||
ColumnName string `gorm:"column:COLUMN_NAME"`
|
||||
}
|
||||
var pkColumns []pkInfo
|
||||
if err := s.db.Raw(`
|
||||
SELECT COLUMN_NAME
|
||||
FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE
|
||||
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND CONSTRAINT_NAME = 'PRIMARY'
|
||||
`, dbName, col.TableName).Scan(&pkColumns).Error; err != nil {
|
||||
logger.Errorf("查询表 %s 的主键信息失败: %v", col.TableName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
hasPK := len(pkColumns) > 0
|
||||
pkOnIdOnly := hasPK && len(pkColumns) == 1 && pkColumns[0].ColumnName == "id"
|
||||
|
||||
needAlter := false
|
||||
switch {
|
||||
case !hasPK:
|
||||
// 没有任何主键:允许将 id 设置为自增主键
|
||||
if col.ColumnKey != "PRI" || col.Extra == "" || !strings.Contains(col.Extra, "auto_increment") {
|
||||
needAlter = true
|
||||
}
|
||||
case pkOnIdOnly:
|
||||
// 只有 id 作为主键:只补充自增属性
|
||||
if col.Extra == "" || !strings.Contains(col.Extra, "auto_increment") {
|
||||
needAlter = true
|
||||
}
|
||||
default:
|
||||
// 已存在非 id 或组合主键:避免破坏原有主键,直接跳过
|
||||
logger.Infof("表 %s 已存在非 id 主键,跳过 id 自增主键修复", col.TableName)
|
||||
}
|
||||
if !needAlter {
|
||||
continue
|
||||
}
|
||||
|
||||
// 维持原来的数据类型,避免与历史 SQL 冲突
|
||||
dataType := col.DataType
|
||||
if dataType == "" {
|
||||
dataType = "int"
|
||||
}
|
||||
|
||||
alterSQL := fmt.Sprintf(
|
||||
"ALTER TABLE `%s` MODIFY COLUMN id %s NOT NULL AUTO_INCREMENT PRIMARY KEY",
|
||||
col.TableName,
|
||||
dataType,
|
||||
)
|
||||
if err := s.db.Exec(alterSQL).Error; err != nil {
|
||||
logger.Errorf("修复表 %s 的 id 自增主键失败: %v", col.TableName, err)
|
||||
} else {
|
||||
logger.Infof("已修复表 %s 的 id 为 AUTO_INCREMENT PRIMARY KEY", col.TableName)
|
||||
}
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
|
||||
// === 为 geekai_users.username 补充唯一索引(根据模型 uniqueIndex 定义) ===
|
||||
var usernameUniqueCount int64
|
||||
err := s.db.Raw(`
|
||||
SELECT COUNT(1)
|
||||
FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = ?
|
||||
AND TABLE_NAME = 'geekai_users'
|
||||
AND COLUMN_NAME = 'username'
|
||||
AND NON_UNIQUE = 0
|
||||
`, dbName).Scan(&usernameUniqueCount).Error
|
||||
if err != nil {
|
||||
logger.Errorf("检查 geekai_users.username 唯一索引失败: %v", err)
|
||||
} else if usernameUniqueCount == 0 {
|
||||
// 索引名尽量固定,避免重复创建
|
||||
if err := s.db.Exec("ALTER TABLE geekai_users ADD UNIQUE KEY idx_geekai_users_username (username)").Error; err != nil {
|
||||
logger.Errorf("创建 geekai_users.username 唯一索引失败: %v", err)
|
||||
} else {
|
||||
logger.Info("已为 geekai_users.username 创建唯一索引 idx_geekai_users_username")
|
||||
}
|
||||
} else {
|
||||
logger.Info("geekai_users.username 唯一索引已存在,跳过创建")
|
||||
}
|
||||
|
||||
logger.Info("关键表主键、自增和索引修复完成")
|
||||
}
|
||||
|
||||
func (s *MigrationService) syncModel(dst any) error {
|
||||
tableName := s.tableName(dst)
|
||||
if err := s.db.AutoMigrate(dst); err != nil {
|
||||
return fmt.Errorf("AutoMigrate %s: %w", tableName, err)
|
||||
// incrementalTableMigration 第二步:增量迁移,仅处理删除字段与数据迁移
|
||||
// AutoMigrate 不会删除列,故需在此显式 DropColumn;字段重命名需先拷贝数据再删除旧列
|
||||
func (s *MigrationService) incrementalTableMigration() {
|
||||
logger.Info("执行增量迁移(删除字段 + 数据迁移)...")
|
||||
|
||||
// ========== 字段重命名:全量迁移已添加新列,需将旧列数据拷贝到新列后删除旧列 ==========
|
||||
|
||||
// ChatApp(geekai_chat_roles): context_json -> system_prompt 历史数据迁移
|
||||
if s.db.Migrator().HasColumn(&model.ChatApp{}, "context_json") {
|
||||
// 将旧列 context_json 的值拷贝到 system_prompt(NULL 转为空字符串,保证 NOT NULL 约束)
|
||||
if err := s.db.Exec(`
|
||||
UPDATE geekai_chat_roles
|
||||
SET system_prompt = IFNULL(NULLIF(TRIM(COALESCE(context_json, '')), ''), '')
|
||||
`).Error; err != nil {
|
||||
logger.Errorf("迁移 geekai_chat_roles.context_json -> system_prompt 失败: %v", err)
|
||||
} else {
|
||||
if err := s.db.Migrator().DropColumn(&model.ChatApp{}, "context_json"); err != nil {
|
||||
logger.Errorf("删除 geekai_chat_roles.context_json 失败: %v", err)
|
||||
} else {
|
||||
logger.Info("geekai_chat_roles: context_json 已迁移至 system_prompt 并删除旧列")
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := s.dropUnusedColumns(dst); err != nil {
|
||||
return fmt.Errorf("drop unused columns %s: %w", tableName, err)
|
||||
|
||||
// ChatApp: 将 user_id 为 NULL 的历史记录置为 0(系统内置)
|
||||
if s.db.Migrator().HasColumn(&model.ChatApp{}, "user_id") {
|
||||
if err := s.db.Exec(`UPDATE geekai_chat_roles SET user_id = 0 WHERE user_id IS NULL`).Error; err != nil {
|
||||
logger.Errorf("初始化 geekai_chat_roles.user_id 失败: %v", err)
|
||||
}
|
||||
}
|
||||
logger.Infof("已同步数据表: %s", tableName)
|
||||
return nil
|
||||
|
||||
// ChatApp(geekai_chat_roles): 删除 marker 列(应用仅通过 id 区分)
|
||||
var hasMarker int
|
||||
if s.db.Raw("SELECT COUNT(1) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'geekai_chat_roles' AND COLUMN_NAME = 'marker'").Scan(&hasMarker).Error == nil && hasMarker > 0 {
|
||||
// 先删除可能存在的唯一索引(不同版本 SQL 索引名不同)
|
||||
for _, idxName := range []string{"marker", "idx_chatgpt_chat_roles_marker", "idx_chatgpt_chat_roles_key", "idx_geekai_chat_roles_marker"} {
|
||||
_ = s.db.Exec(fmt.Sprintf("ALTER TABLE geekai_chat_roles DROP INDEX `%s`", idxName)).Error
|
||||
}
|
||||
if err := s.db.Exec("ALTER TABLE geekai_chat_roles DROP COLUMN marker").Error; err != nil {
|
||||
logger.Errorf("删除 geekai_chat_roles.marker 失败: %v", err)
|
||||
} else {
|
||||
logger.Info("geekai_chat_roles: 已删除 marker 列")
|
||||
}
|
||||
}
|
||||
|
||||
// Config: config_json -> value, marker -> name
|
||||
if s.db.Migrator().HasColumn(&model.Config{}, "config_json") {
|
||||
s.db.Exec("UPDATE geekai_configs SET `value` = config_json WHERE config_json IS NOT NULL AND config_json != ''")
|
||||
s.db.Migrator().DropColumn(&model.Config{}, "config_json")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.Config{}, "marker") {
|
||||
s.db.Exec("UPDATE geekai_configs SET `name` = marker WHERE marker IS NOT NULL AND marker != ''")
|
||||
s.db.Migrator().DropColumn(&model.Config{}, "marker")
|
||||
}
|
||||
if s.db.Migrator().HasIndex(&model.Config{}, "idx_chatgpt_configs_key") {
|
||||
s.db.Migrator().DropIndex(&model.Config{}, "idx_chatgpt_configs_key")
|
||||
}
|
||||
if s.db.Migrator().HasIndex(&model.Config{}, "marker") {
|
||||
s.db.Migrator().DropIndex(&model.Config{}, "marker")
|
||||
}
|
||||
|
||||
// Order: pay_type -> channel
|
||||
if s.db.Migrator().HasColumn(&model.Order{}, "pay_type") {
|
||||
s.db.Exec("UPDATE geekai_orders SET channel = pay_type WHERE pay_type IS NOT NULL AND pay_type != ''")
|
||||
s.db.Migrator().DropColumn(&model.Order{}, "pay_type")
|
||||
}
|
||||
|
||||
// JimengJob: task_params -> params
|
||||
if s.db.Migrator().HasColumn(&model.JimengJob{}, "task_params") {
|
||||
s.db.Exec("UPDATE geekai_jimeng_jobs SET params = task_params WHERE task_params IS NOT NULL AND task_params != ''")
|
||||
s.db.Migrator().DropColumn(&model.JimengJob{}, "task_params")
|
||||
}
|
||||
|
||||
// VideoJob: task_info -> params, raw_data -> output
|
||||
if s.db.Migrator().HasColumn(&model.VideoJob{}, "task_info") {
|
||||
s.db.Exec("UPDATE geekai_video_jobs SET params = task_info WHERE task_info IS NOT NULL AND task_info != ''")
|
||||
s.db.Migrator().DropColumn(&model.VideoJob{}, "task_info")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.VideoJob{}, "raw_data") {
|
||||
s.db.Exec("UPDATE geekai_video_jobs SET `output` = raw_data WHERE raw_data IS NOT NULL AND raw_data != ''")
|
||||
s.db.Migrator().DropColumn(&model.VideoJob{}, "raw_data")
|
||||
}
|
||||
|
||||
// SunoJob: task_info -> params, raw_data -> output
|
||||
if s.db.Migrator().HasColumn(&model.SunoJob{}, "task_info") {
|
||||
s.db.Exec("UPDATE geekai_suno_jobs SET params = task_info WHERE task_info IS NOT NULL AND task_info != ''")
|
||||
s.db.Migrator().DropColumn(&model.SunoJob{}, "task_info")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.SunoJob{}, "raw_data") {
|
||||
s.db.Exec("UPDATE geekai_suno_jobs SET `output` = raw_data WHERE raw_data IS NOT NULL AND raw_data != ''")
|
||||
s.db.Migrator().DropColumn(&model.SunoJob{}, "raw_data")
|
||||
}
|
||||
|
||||
// ========== 删除不再使用的字段 ==========
|
||||
|
||||
if s.db.Migrator().HasColumn(&model.Order{}, "deleted_at") {
|
||||
s.db.Migrator().DropColumn(&model.Order{}, "deleted_at")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.ChatItem{}, "deleted_at") {
|
||||
s.db.Migrator().DropColumn(&model.ChatItem{}, "deleted_at")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.ChatMessage{}, "deleted_at") {
|
||||
s.db.Migrator().DropColumn(&model.ChatMessage{}, "deleted_at")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.User{}, "chat_config") {
|
||||
s.db.Migrator().DropColumn(&model.User{}, "chat_config")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.ChatModel{}, "category") {
|
||||
s.db.Migrator().DropColumn(&model.ChatModel{}, "category")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.ChatModel{}, "description") {
|
||||
s.db.Migrator().DropColumn(&model.ChatModel{}, "description")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.Product{}, "discount") {
|
||||
s.db.Migrator().DropColumn(&model.Product{}, "discount")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.Product{}, "days") {
|
||||
s.db.Migrator().DropColumn(&model.Product{}, "days")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.Product{}, "app_url") {
|
||||
s.db.Migrator().DropColumn(&model.Product{}, "app_url")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.Product{}, "url") {
|
||||
s.db.Migrator().DropColumn(&model.Product{}, "url")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.VideoJob{}, "water_url") {
|
||||
s.db.Migrator().DropColumn(&model.VideoJob{}, "water_url")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.VideoJob{}, "cover_url") {
|
||||
s.db.Migrator().DropColumn(&model.VideoJob{}, "cover_url")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.VideoJob{}, "prompt_ext") {
|
||||
s.db.Migrator().DropColumn(&model.VideoJob{}, "prompt_ext")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.SunoJob{}, "instrumental") {
|
||||
s.db.Migrator().DropColumn(&model.SunoJob{}, "instrumental")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.SunoJob{}, "tags") {
|
||||
s.db.Migrator().DropColumn(&model.SunoJob{}, "tags")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.SunoJob{}, "extend_secs") {
|
||||
s.db.Migrator().DropColumn(&model.SunoJob{}, "extend_secs")
|
||||
}
|
||||
if s.db.Migrator().HasColumn(&model.SunoJob{}, "model_name") {
|
||||
s.db.Migrator().DropColumn(&model.SunoJob{}, "model_name")
|
||||
}
|
||||
|
||||
// ========== 数据迁移:根据业务逻辑更新现有数据 ==========
|
||||
|
||||
// video_job: 根据 progress 填充 status
|
||||
if s.db.Migrator().HasColumn(&model.VideoJob{}, "status") {
|
||||
s.db.Exec(`UPDATE geekai_video_jobs SET status = CASE
|
||||
WHEN progress < 100 THEN 'in_progress'
|
||||
WHEN progress = 100 THEN 'success'
|
||||
WHEN progress = 101 THEN 'failed'
|
||||
WHEN progress = 102 THEN 'downloading'
|
||||
ELSE 'pending'
|
||||
END WHERE status = '' OR status IS NULL`)
|
||||
}
|
||||
|
||||
// suno_job: 从 output 提取 tags/model_name 填入 params
|
||||
s.migrateSunoJobData()
|
||||
|
||||
logger.Info("增量迁移完成")
|
||||
}
|
||||
|
||||
func (s *MigrationService) dropUnusedColumns(dst any) error {
|
||||
dbCols, err := s.db.Migrator().ColumnTypes(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
// TableMigration 数据表迁移入口:先全量同步 schema,再增量删除字段并迁移数据
|
||||
func (s *MigrationService) TableMigration() {
|
||||
s.fullTableMigration()
|
||||
s.fixTableConstraints()
|
||||
s.incrementalTableMigration()
|
||||
s.migrateChatAppSystemPromptFromJSON()
|
||||
}
|
||||
|
||||
// migrateChatAppSystemPromptFromJSON 将智能体 system_prompt 字段中历史 JSON 数组
|
||||
// 解析后取出 role 为 system 的 content,覆盖回 system_prompt(纯文本)
|
||||
func (s *MigrationService) migrateChatAppSystemPromptFromJSON() {
|
||||
key := "migrate:chat_app_system_prompt_json"
|
||||
if s.redisClient.Get(context.Background(), key).Val() == "1" {
|
||||
logger.Info("ChatApp system_prompt JSON 已迁移,跳过")
|
||||
return
|
||||
}
|
||||
modelCols, err := s.modelColumnNames(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
logger.Info("开始迁移智能体 system_prompt 历史 JSON 数据...")
|
||||
|
||||
var apps []model.ChatApp
|
||||
if err := s.db.Find(&apps).Error; err != nil {
|
||||
logger.Errorf("查询 ChatApp 失败: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, col := range dbCols {
|
||||
name := col.Name()
|
||||
if s.isProtectedColumn(name) {
|
||||
updated := 0
|
||||
for i := range apps {
|
||||
raw := strings.TrimSpace(apps[i].SystemPrompt)
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := modelCols[strings.ToLower(name)]; ok {
|
||||
if len(raw) < 2 || raw[0] != '[' {
|
||||
continue
|
||||
}
|
||||
logger.Infof("删除多余字段: %s.%s", s.tableName(dst), name)
|
||||
if err := s.db.Migrator().DropColumn(dst, name); err != nil {
|
||||
return fmt.Errorf("DropColumn %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MigrationService) modelColumnNames(dst any) (map[string]struct{}, error) {
|
||||
parsed, err := schema.Parse(dst, &schemaCache, s.db.Config.NamingStrategy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cols := make(map[string]struct{}, len(parsed.Fields))
|
||||
for _, field := range parsed.Fields {
|
||||
if field.DBName == "" || field.IgnoreMigration {
|
||||
var messages []types.Message
|
||||
if err := json.Unmarshal([]byte(raw), &messages); err != nil {
|
||||
continue
|
||||
}
|
||||
cols[strings.ToLower(field.DBName)] = struct{}{}
|
||||
var systemContent string
|
||||
for _, m := range messages {
|
||||
if strings.ToLower(strings.TrimSpace(m.Role)) == "system" && m.Content != "" {
|
||||
systemContent = m.Content
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := s.db.Model(&model.ChatApp{}).Where("id = ?", apps[i].Id).Update("system_prompt", systemContent).Error; err != nil {
|
||||
logger.Warnf("更新 ChatApp id=%d system_prompt 失败: %v", apps[i].Id, err)
|
||||
continue
|
||||
}
|
||||
updated++
|
||||
}
|
||||
return cols, nil
|
||||
}
|
||||
|
||||
func (s *MigrationService) isProtectedColumn(name string) bool {
|
||||
_, ok := protectedColumns[strings.ToLower(name)]
|
||||
return ok
|
||||
logger.Infof("智能体 system_prompt JSON 迁移完成,共更新 %d 条", updated)
|
||||
s.redisClient.Set(context.Background(), key, "1", 0)
|
||||
}
|
||||
|
||||
func (s *MigrationService) tableName(dst any) string {
|
||||
stmt := &gorm.Statement{DB: s.db}
|
||||
if err := stmt.Parse(dst); err != nil {
|
||||
return fmt.Sprintf("%T", dst)
|
||||
}
|
||||
return stmt.Schema.Table
|
||||
}
|
||||
|
||||
// schema.Parse 进程内复用的 schema cache
|
||||
var schemaCache sync.Map
|
||||
|
||||
// 迁移配置数据
|
||||
func (s *MigrationService) MigrateConfig(config *types.AppConfig) error {
|
||||
|
||||
@@ -374,6 +593,14 @@ func (s *MigrationService) migrateCommunicationConfig(config *types.AppConfig) e
|
||||
"sign": config.SMS.Bao.Sign,
|
||||
"code_template": config.SMS.Bao.CodeTemplate,
|
||||
},
|
||||
"tencent": map[string]any{
|
||||
"secret_id": config.SMS.Tencent.SecretId,
|
||||
"secret_key": config.SMS.Tencent.SecretKey,
|
||||
"sms_sdk_app_id": config.SMS.Tencent.SmsSdkAppId,
|
||||
"sign": config.SMS.Tencent.Sign,
|
||||
"code_temp_id": config.SMS.Tencent.CodeTempId,
|
||||
"region": config.SMS.Tencent.Region,
|
||||
},
|
||||
}
|
||||
return s.saveConfig(types.ConfigKeySms, smsConfig)
|
||||
}
|
||||
@@ -406,3 +633,73 @@ func (s *MigrationService) saveConfig(key string, config any) error {
|
||||
logger.Infof("成功迁移配置 %s", key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateSunoJobData 合并 suno_job 数据:从 Output 原始数据中解析出 tags 和 model_name 填入 params 字段
|
||||
func (s *MigrationService) migrateSunoJobData() {
|
||||
key := "migrate:suno_job_data"
|
||||
if s.redisClient.Get(context.Background(), key).Val() == "1" {
|
||||
logger.Info("SunoJob 数据已合并,跳过迁移")
|
||||
return
|
||||
}
|
||||
|
||||
logger.Info("开始合并 SunoJob 数据...")
|
||||
|
||||
// 查询所有有 output 数据的记录
|
||||
var jobs []model.SunoJob
|
||||
if err := s.db.Where("output != ? AND output != ''", "").Find(&jobs).Error; err != nil {
|
||||
logger.Errorf("查询 SunoJob 数据失败: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
updatedCount := 0
|
||||
for _, job := range jobs {
|
||||
if job.Output == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// 解析 Output JSON 数据
|
||||
var outputData struct {
|
||||
Metadata struct {
|
||||
Tags string `json:"tags"`
|
||||
} `json:"metadata"`
|
||||
ModelName string `json:"model_name"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(job.Output), &outputData); err != nil {
|
||||
logger.Warnf("解析 Output 数据失败 (ID: %d): %v", job.Id, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// 检查是否需要更新 params
|
||||
needUpdate := false
|
||||
params := job.Params
|
||||
|
||||
// 如果 params 中的 tags 为空,但 output 中有 tags,则更新
|
||||
if params.Tags == "" && outputData.Metadata.Tags != "" {
|
||||
params.Tags = outputData.Metadata.Tags
|
||||
// 修复 tags 字段过长导致更新失败
|
||||
if len(params.Tags) > 255 {
|
||||
params.Tags = params.Tags[:255]
|
||||
}
|
||||
needUpdate = true
|
||||
}
|
||||
|
||||
// 如果 params 中的 model 为空,但 output 中有 model_name,则更新
|
||||
if params.Model == "" && outputData.ModelName != "" {
|
||||
params.Model = outputData.ModelName
|
||||
needUpdate = true
|
||||
}
|
||||
|
||||
// 如果需要更新,则保存
|
||||
if needUpdate {
|
||||
if err := s.db.Model(&model.SunoJob{}).Where("id = ?", job.Id).Update("params", params).Error; err != nil {
|
||||
logger.Errorf("更新 SunoJob 数据失败 (ID: %d): %v", job.Id, err)
|
||||
continue
|
||||
}
|
||||
updatedCount++
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("SunoJob 数据合并完成,共更新 %d 条记录", updatedCount)
|
||||
s.redisClient.Set(context.Background(), key, "1", 0)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user