chore: v3 config migration script

pkg/core/app.py

@@ -67,8 +67,6 @@ class Application:
 
     sensitive_meta: config_mgr.ConfigManager = None
 
-    instance_secret_meta: config_mgr.ConfigManager = None  # deprecated
-
     pipeline_config_meta_trigger: config_mgr.ConfigManager = None
     pipeline_config_meta_safety: config_mgr.ConfigManager = None
     pipeline_config_meta_ai: config_mgr.ConfigManager = None

pkg/core/stages/build_app.py

@@ -51,8 +51,8 @@ class BuildAppStage(stage.BootingStage):
         ap.log_cache = log_cache
 
         persistence_mgr_inst = persistencemgr.PersistenceManager(ap)
-        await persistence_mgr_inst.initialize()
         ap.persistence_mgr = persistence_mgr_inst
+        await persistence_mgr_inst.initialize()
 
         plugin_mgr_inst = plugin_mgr.PluginManager(ap)
         await plugin_mgr_inst.initialize()

pkg/core/stages/load_config.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import secrets
 import os
 
 from .. import stage, app
@@ -50,12 +49,6 @@ class LoadConfigStage(stage.BootingStage):
                 completion=False,
             )
 
-        if os.path.exists('data/metadata/instance-secret.json'):
-            ap.instance_secret_meta = await config.load_json_config(
-                'data/metadata/instance-secret.json',
-                template_data={'jwt_secret': secrets.token_hex(16)},
-            )
-            await ap.instance_secret_meta.dump_config()
         # ======= deprecated =======
 
         ap.instance_config = await config.load_yaml_config(

pkg/persistence/mgr.py

@@ -76,7 +76,7 @@ class PersistenceManager:
                 'for_version': self.ap.ver_mgr.get_current_version(),
                 'stages': pipeline_service.default_stage_order,
                 'is_default': True,
-                'name': 'Chat Pipeline',
+                'name': 'ChatPipeline',
                 'description': '默认提供的流水线，您配置的机器人、第一个模型将自动绑定到此流水线',
                 'config': pipeline_config,
             }

pkg/persistence/migrations/dbm001_migrate_v3_config.py

@@ -1,11 +1,234 @@
-# TODO fill this
+from .. import migration
-# @migration.migration_class(1)
+from copy import deepcopy
-# class DBMigrationV3(migration.DBMigration):
+import uuid
-#     """数据库迁移"""
+import os
+import sqlalchemy
+import shutil
 
-#     async def upgrade(self):
+from ...config import manager as config_manager
-#         """升级"""
+from ...entity.persistence import (
-#         pass
+    model as persistence_model,
+    pipeline as persistence_pipeline,
+    bot as persistence_bot,
+)
 
-#     async def downgrade(self):
+
-#         """降级"""
+@migration.migration_class(1)
+class DBMigrateV3Config(migration.DBMigration):
+    """从 v3 的配置迁移到 v4 的数据库"""
+
+    async def upgrade(self):
+        """升级"""
+        """
+        将 data/config 下的所有配置文件进行迁移。
+        迁移后，之前的配置文件都保存到 data/legacy/config 下。
+        迁移后，data/metadata/ 下的所有配置文件都保存到 data/legacy/metadata 下。
+        """
+
+        if self.ap.provider_cfg is None:
+            return
+
+        # ======= 迁移模型 =======
+        # 只迁移当前选中的模型
+        model_name = self.ap.provider_cfg.data.get('model', 'gpt-4o')
+
+        model_requester = 'openai-chat-completions'
+        model_requester_config = {}
+        model_api_keys = ['sk-proj-1234567890']
+        model_abilities = []
+        model_extra_args = {}
+
+        if os.path.exists('data/metadata/llm-models.json'):
+            _llm_model_meta = await config_manager.load_json_config('data/metadata/llm-models.json', completion=False)
+
+            for item in _llm_model_meta.data.get('list', []):
+                if item.get('name') == model_name:
+                    if 'model_name' in item:
+                        model_name = item['model_name']
+                    if 'requester' in item:
+                        model_requester = item['requester']
+                    if 'token_mgr' in item:
+                        _token_mgr = item['token_mgr']
+
+                        if _token_mgr in self.ap.provider_cfg.data.get('keys', {}):
+                            model_api_keys = self.ap.provider_cfg.data.get('keys', {})[_token_mgr]
+
+                    if 'tool_call_supported' in item and item['tool_call_supported']:
+                        model_abilities.append(item['func_call'])
+
+                    if 'vision_supported' in item and item['vision_supported']:
+                        model_abilities.append('vision')
+
+                    if (
+                        model_requester in self.ap.provider_cfg.data.get('requester', {})
+                        and 'args' in self.ap.provider_cfg.data.get('requester', {})[model_requester]
+                    ):
+                        model_extra_args = self.ap.provider_cfg.data.get('requester', {})[model_requester]['args']
+
+                    if model_requester in self.ap.provider_cfg.data.get('requester', {}):
+                        model_requester_config = self.ap.provider_cfg.data.get('requester', {})[model_requester]
+                        model_requester_config = {
+                            'base_url': model_requester_config['base-url'],
+                            'timeout': model_requester_config['timeout'],
+                        }
+
+                    break
+
+        model_uuid = str(uuid.uuid4())
+
+        llm_model_data = {
+            'uuid': model_uuid,
+            'name': model_name,
+            'description': '由 LangBot v3 迁移而来',
+            'requester': model_requester,
+            'requester_config': model_requester_config,
+            'api_keys': model_api_keys,
+            'abilities': model_abilities,
+            'extra_args': model_extra_args,
+        }
+
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.insert(persistence_model.LLMModel).values(**llm_model_data)
+        )
+
+        # ======= 迁移流水线配置 =======
+        # 修改到默认流水线
+        default_pipeline = [
+            self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
+            for pipeline in (
+                await self.ap.persistence_mgr.execute_async(
+                    sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
+                        persistence_pipeline.LegacyPipeline.is_default == True
+                    )
+                )
+            ).all()
+        ][0]
+
+        pipeline_uuid = str(uuid.uuid4())
+        pipeline_name = 'ChatPipeline'
+
+        if default_pipeline:
+            pipeline_name = default_pipeline['name']
+            pipeline_uuid = default_pipeline['uuid']
+
+            pipeline_config = default_pipeline['config']
+
+            # ai
+            pipeline_config['ai']['runner'] = {
+                'runner': self.ap.provider_cfg.data['runner'],
+            }
+            pipeline_config['ai']['local-agent']['model'] = model_uuid
+            pipeline_config['ai']['local-agent']['max-round'] = self.ap.pipeline_cfg.data['msg-truncate']['round'][
+                'max-round'
+            ]
+
+            pipeline_config['ai']['local-agent']['prompt'] = [
+                {
+                    'role': 'system',
+                    'content': self.ap.provider_cfg.data['prompt']['default'],
+                }
+            ]
+            pipeline_config['ai']['dify-service-api'] = {
+                'base-url': self.ap.provider_cfg.data['dify-service-api']['base-url'],
+                'app-type': self.ap.provider_cfg.data['dify-service-api']['app-type'],
+                'api-key': self.ap.provider_cfg.data['dify-service-api'][
+                    self.ap.provider_cfg.data['dify-service-api']['app-type']
+                ]['api-key'],
+                'thinking-convert': self.ap.provider_cfg.data['dify-service-api']['options']['convert-thinking-tips'],
+                'timeout': self.ap.provider_cfg.data['dify-service-api'][
+                    self.ap.provider_cfg.data['dify-service-api']['app-type']
+                ]['timeout'],
+            }
+            pipeline_config['ai']['dashscope-app-api'] = {
+                'app-type': self.ap.provider_cfg.data['dashscope-app-api']['app-type'],
+                'api-key': self.ap.provider_cfg.data['dashscope-app-api']['api-key'],
+                'references_quote': self.ap.provider_cfg.data['dashscope-app-api'][
+                    self.ap.provider_cfg.data['dashscope-app-api']['app-type']
+                ]['references_quote'],
+            }
+
+            # trigger
+            pipeline_config['trigger']['group-respond-rules'] = self.ap.pipeline_cfg.data['respond-rules']['default']
+            pipeline_config['trigger']['access-control'] = self.ap.pipeline_cfg.data['access-control']
+            pipeline_config['trigger']['ignore-rules'] = self.ap.pipeline_cfg.data['ignore-rules']
+
+            # safety
+            pipeline_config['safety']['content-filter'] = {
+                'scope': 'all',
+                'check-sensitive-words': self.ap.pipeline_cfg.data['check-sensitive-words'],
+            }
+            pipeline_config['safety']['rate-limit'] = {
+                'window-length': self.ap.pipeline_cfg.data['rate-limit']['fixwin']['default']['window-size'],
+                'limitation': self.ap.pipeline_cfg.data['rate-limit']['fixwin']['default']['limit'],
+                'strategy': self.ap.pipeline_cfg.data['rate-limit']['strategy'],
+            }
+
+            # output
+            pipeline_config['output']['long-text-processing'] = self.ap.platform_cfg.data['long-text-process']
+            pipeline_config['output']['force-delay'] = self.ap.platform_cfg.data['force-delay']
+            pipeline_config['output']['misc'] = {
+                'hide-exception': self.ap.platform_cfg.data['hide-exception-info'],
+                'quote-origin': self.ap.platform_cfg.data['quote-origin'],
+                'at-sender': self.ap.platform_cfg.data['at-sender'],
+                'track-function-calls': self.ap.platform_cfg.data['track-function-calls'],
+            }
+
+            default_pipeline['description'] = default_pipeline['description'] + ' [已迁移 LangBot v3 配置]'
+            default_pipeline['config'] = pipeline_config
+            default_pipeline.pop('created_at')
+            default_pipeline.pop('updated_at')
+
+            await self.ap.persistence_mgr.execute_async(
+                sqlalchemy.update(persistence_pipeline.LegacyPipeline)
+                .values(default_pipeline)
+                .where(persistence_pipeline.LegacyPipeline.uuid == default_pipeline['uuid'])
+            )
+
+        # ======= 迁移机器人 =======
+        # 只迁移启用的机器人
+        for adapter in self.ap.platform_cfg.data.get('platform-adapters', []):
+            if not adapter.get('enable'):
+                continue
+
+            args = deepcopy(adapter)
+            args.pop('adapter')
+            args.pop('enable')
+
+            bot_data = {
+                'uuid': str(uuid.uuid4()),
+                'name': adapter.get('adapter'),
+                'description': '由 LangBot v3 迁移而来',
+                'adapter': adapter.get('adapter'),
+                'adapter_config': args,
+                'enable': True,
+                'use_pipeline_uuid': pipeline_uuid,
+                'use_pipeline_name': pipeline_name,
+            }
+
+            await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_bot.Bot).values(**bot_data))
+
+        # ======= move files =======
+        # 迁移 data/config 下的所有配置文件
+        all_legacy_dir_name = [
+            'config',
+            # 'metadata',
+            'prompts',
+            'scenario',
+        ]
+
+        def move_legacy_files(dir_name: str):
+            if not os.path.exists(f'data/legacy/{dir_name}'):
+                os.makedirs(f'data/legacy/{dir_name}')
+
+            if os.path.exists(f'data/{dir_name}'):
+                for file in os.listdir(f'data/{dir_name}'):
+                    if file.endswith('.json'):
+                        shutil.move(f'data/{dir_name}/{file}', f'data/legacy/{dir_name}/{file}')
+
+                os.rmdir(f'data/{dir_name}')
+
+        for dir_name in all_legacy_dir_name:
+            move_legacy_files(dir_name)
+
+    async def downgrade(self):
+        """降级"""