diff --git a/pkg/core/app.py b/pkg/core/app.py index 692e2b8a..ad1d45c7 100644 --- a/pkg/core/app.py +++ b/pkg/core/app.py @@ -67,8 +67,6 @@ class Application: sensitive_meta: config_mgr.ConfigManager = None - instance_secret_meta: config_mgr.ConfigManager = None # deprecated - pipeline_config_meta_trigger: config_mgr.ConfigManager = None pipeline_config_meta_safety: config_mgr.ConfigManager = None pipeline_config_meta_ai: config_mgr.ConfigManager = None diff --git a/pkg/core/stages/build_app.py b/pkg/core/stages/build_app.py index 3bebea00..2cce2bc5 100644 --- a/pkg/core/stages/build_app.py +++ b/pkg/core/stages/build_app.py @@ -51,8 +51,8 @@ class BuildAppStage(stage.BootingStage): ap.log_cache = log_cache persistence_mgr_inst = persistencemgr.PersistenceManager(ap) - await persistence_mgr_inst.initialize() ap.persistence_mgr = persistence_mgr_inst + await persistence_mgr_inst.initialize() plugin_mgr_inst = plugin_mgr.PluginManager(ap) await plugin_mgr_inst.initialize() diff --git a/pkg/core/stages/load_config.py b/pkg/core/stages/load_config.py index ac2f0c37..ef5f611b 100644 --- a/pkg/core/stages/load_config.py +++ b/pkg/core/stages/load_config.py @@ -1,6 +1,5 @@ from __future__ import annotations -import secrets import os from .. import stage, app @@ -50,12 +49,6 @@ class LoadConfigStage(stage.BootingStage): completion=False, ) - if os.path.exists('data/metadata/instance-secret.json'): - ap.instance_secret_meta = await config.load_json_config( - 'data/metadata/instance-secret.json', - template_data={'jwt_secret': secrets.token_hex(16)}, - ) - await ap.instance_secret_meta.dump_config() # ======= deprecated ======= ap.instance_config = await config.load_yaml_config( diff --git a/pkg/persistence/mgr.py b/pkg/persistence/mgr.py index 69382e6d..f0d7459b 100644 --- a/pkg/persistence/mgr.py +++ b/pkg/persistence/mgr.py @@ -76,7 +76,7 @@ class PersistenceManager: 'for_version': self.ap.ver_mgr.get_current_version(), 'stages': pipeline_service.default_stage_order, 'is_default': True, - 'name': 'Chat Pipeline', + 'name': 'ChatPipeline', 'description': '默认提供的流水线，您配置的机器人、第一个模型将自动绑定到此流水线', 'config': pipeline_config, } diff --git a/pkg/persistence/migrations/dbm001_migrate_v3_config.py b/pkg/persistence/migrations/dbm001_migrate_v3_config.py index 6aee2854..21496349 100644 --- a/pkg/persistence/migrations/dbm001_migrate_v3_config.py +++ b/pkg/persistence/migrations/dbm001_migrate_v3_config.py @@ -1,11 +1,234 @@ -# TODO fill this -# @migration.migration_class(1) -# class DBMigrationV3(migration.DBMigration): -# """数据库迁移""" +from .. import migration +from copy import deepcopy +import uuid +import os +import sqlalchemy +import shutil -# async def upgrade(self): -# """升级""" -# pass +from ...config import manager as config_manager +from ...entity.persistence import ( + model as persistence_model, + pipeline as persistence_pipeline, + bot as persistence_bot, +) -# async def downgrade(self): -# """降级""" + +@migration.migration_class(1) +class DBMigrateV3Config(migration.DBMigration): + """从 v3 的配置迁移到 v4 的数据库""" + + async def upgrade(self): + """升级""" + """ + 将 data/config 下的所有配置文件进行迁移。 + 迁移后，之前的配置文件都保存到 data/legacy/config 下。 + 迁移后，data/metadata/ 下的所有配置文件都保存到 data/legacy/metadata 下。 + """ + + if self.ap.provider_cfg is None: + return + + # ======= 迁移模型 ======= + # 只迁移当前选中的模型 + model_name = self.ap.provider_cfg.data.get('model', 'gpt-4o') + + model_requester = 'openai-chat-completions' + model_requester_config = {} + model_api_keys = ['sk-proj-1234567890'] + model_abilities = [] + model_extra_args = {} + + if os.path.exists('data/metadata/llm-models.json'): + _llm_model_meta = await config_manager.load_json_config('data/metadata/llm-models.json', completion=False) + + for item in _llm_model_meta.data.get('list', []): + if item.get('name') == model_name: + if 'model_name' in item: + model_name = item['model_name'] + if 'requester' in item: + model_requester = item['requester'] + if 'token_mgr' in item: + _token_mgr = item['token_mgr'] + + if _token_mgr in self.ap.provider_cfg.data.get('keys', {}): + model_api_keys = self.ap.provider_cfg.data.get('keys', {})[_token_mgr] + + if 'tool_call_supported' in item and item['tool_call_supported']: + model_abilities.append(item['func_call']) + + if 'vision_supported' in item and item['vision_supported']: + model_abilities.append('vision') + + if ( + model_requester in self.ap.provider_cfg.data.get('requester', {}) + and 'args' in self.ap.provider_cfg.data.get('requester', {})[model_requester] + ): + model_extra_args = self.ap.provider_cfg.data.get('requester', {})[model_requester]['args'] + + if model_requester in self.ap.provider_cfg.data.get('requester', {}): + model_requester_config = self.ap.provider_cfg.data.get('requester', {})[model_requester] + model_requester_config = { + 'base_url': model_requester_config['base-url'], + 'timeout': model_requester_config['timeout'], + } + + break + + model_uuid = str(uuid.uuid4()) + + llm_model_data = { + 'uuid': model_uuid, + 'name': model_name, + 'description': '由 LangBot v3 迁移而来', + 'requester': model_requester, + 'requester_config': model_requester_config, + 'api_keys': model_api_keys, + 'abilities': model_abilities, + 'extra_args': model_extra_args, + } + + await self.ap.persistence_mgr.execute_async( + sqlalchemy.insert(persistence_model.LLMModel).values(**llm_model_data) + ) + + # ======= 迁移流水线配置 ======= + # 修改到默认流水线 + default_pipeline = [ + self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline) + for pipeline in ( + await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(persistence_pipeline.LegacyPipeline).where( + persistence_pipeline.LegacyPipeline.is_default == True + ) + ) + ).all() + ][0] + + pipeline_uuid = str(uuid.uuid4()) + pipeline_name = 'ChatPipeline' + + if default_pipeline: + pipeline_name = default_pipeline['name'] + pipeline_uuid = default_pipeline['uuid'] + + pipeline_config = default_pipeline['config'] + + # ai + pipeline_config['ai']['runner'] = { + 'runner': self.ap.provider_cfg.data['runner'], + } + pipeline_config['ai']['local-agent']['model'] = model_uuid + pipeline_config['ai']['local-agent']['max-round'] = self.ap.pipeline_cfg.data['msg-truncate']['round'][ + 'max-round' + ] + + pipeline_config['ai']['local-agent']['prompt'] = [ + { + 'role': 'system', + 'content': self.ap.provider_cfg.data['prompt']['default'], + } + ] + pipeline_config['ai']['dify-service-api'] = { + 'base-url': self.ap.provider_cfg.data['dify-service-api']['base-url'], + 'app-type': self.ap.provider_cfg.data['dify-service-api']['app-type'], + 'api-key': self.ap.provider_cfg.data['dify-service-api'][ + self.ap.provider_cfg.data['dify-service-api']['app-type'] + ]['api-key'], + 'thinking-convert': self.ap.provider_cfg.data['dify-service-api']['options']['convert-thinking-tips'], + 'timeout': self.ap.provider_cfg.data['dify-service-api'][ + self.ap.provider_cfg.data['dify-service-api']['app-type'] + ]['timeout'], + } + pipeline_config['ai']['dashscope-app-api'] = { + 'app-type': self.ap.provider_cfg.data['dashscope-app-api']['app-type'], + 'api-key': self.ap.provider_cfg.data['dashscope-app-api']['api-key'], + 'references_quote': self.ap.provider_cfg.data['dashscope-app-api'][ + self.ap.provider_cfg.data['dashscope-app-api']['app-type'] + ]['references_quote'], + } + + # trigger + pipeline_config['trigger']['group-respond-rules'] = self.ap.pipeline_cfg.data['respond-rules']['default'] + pipeline_config['trigger']['access-control'] = self.ap.pipeline_cfg.data['access-control'] + pipeline_config['trigger']['ignore-rules'] = self.ap.pipeline_cfg.data['ignore-rules'] + + # safety + pipeline_config['safety']['content-filter'] = { + 'scope': 'all', + 'check-sensitive-words': self.ap.pipeline_cfg.data['check-sensitive-words'], + } + pipeline_config['safety']['rate-limit'] = { + 'window-length': self.ap.pipeline_cfg.data['rate-limit']['fixwin']['default']['window-size'], + 'limitation': self.ap.pipeline_cfg.data['rate-limit']['fixwin']['default']['limit'], + 'strategy': self.ap.pipeline_cfg.data['rate-limit']['strategy'], + } + + # output + pipeline_config['output']['long-text-processing'] = self.ap.platform_cfg.data['long-text-process'] + pipeline_config['output']['force-delay'] = self.ap.platform_cfg.data['force-delay'] + pipeline_config['output']['misc'] = { + 'hide-exception': self.ap.platform_cfg.data['hide-exception-info'], + 'quote-origin': self.ap.platform_cfg.data['quote-origin'], + 'at-sender': self.ap.platform_cfg.data['at-sender'], + 'track-function-calls': self.ap.platform_cfg.data['track-function-calls'], + } + + default_pipeline['description'] = default_pipeline['description'] + ' [已迁移 LangBot v3 配置]' + default_pipeline['config'] = pipeline_config + default_pipeline.pop('created_at') + default_pipeline.pop('updated_at') + + await self.ap.persistence_mgr.execute_async( + sqlalchemy.update(persistence_pipeline.LegacyPipeline) + .values(default_pipeline) + .where(persistence_pipeline.LegacyPipeline.uuid == default_pipeline['uuid']) + ) + + # ======= 迁移机器人 ======= + # 只迁移启用的机器人 + for adapter in self.ap.platform_cfg.data.get('platform-adapters', []): + if not adapter.get('enable'): + continue + + args = deepcopy(adapter) + args.pop('adapter') + args.pop('enable') + + bot_data = { + 'uuid': str(uuid.uuid4()), + 'name': adapter.get('adapter'), + 'description': '由 LangBot v3 迁移而来', + 'adapter': adapter.get('adapter'), + 'adapter_config': args, + 'enable': True, + 'use_pipeline_uuid': pipeline_uuid, + 'use_pipeline_name': pipeline_name, + } + + await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_bot.Bot).values(**bot_data)) + + # ======= move files ======= + # 迁移 data/config 下的所有配置文件 + all_legacy_dir_name = [ + 'config', + # 'metadata', + 'prompts', + 'scenario', + ] + + def move_legacy_files(dir_name: str): + if not os.path.exists(f'data/legacy/{dir_name}'): + os.makedirs(f'data/legacy/{dir_name}') + + if os.path.exists(f'data/{dir_name}'): + for file in os.listdir(f'data/{dir_name}'): + if file.endswith('.json'): + shutil.move(f'data/{dir_name}/{file}', f'data/legacy/{dir_name}/{file}') + + os.rmdir(f'data/{dir_name}') + + for dir_name in all_legacy_dir_name: + move_legacy_files(dir_name) + + async def downgrade(self): + """降级"""