Merge pull request #729 from RockChinQ/feat/migration-stage

Feat: 配置文件迁移功能

pkg/config/migration.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import abc
+import typing
+
+from ..core import app
+
+
+preregistered_migrations: list[typing.Type[Migration]] = []
+
+def migration_class(name: str, number: int):
+    """注册一个迁移
+    """
+    def decorator(cls: typing.Type[Migration]) -> typing.Type[Migration]:
+        cls.name = name
+        cls.number = number
+        preregistered_migrations.append(cls)
+        return cls
+    
+    return decorator
+
+
+class Migration(abc.ABC):
+    """一个版本的迁移
+    """
+
+    name: str
+
+    number: int
+
+    ap: app.Application
+
+    def __init__(self, ap: app.Application):
+        self.ap = ap
+    
+    @abc.abstractmethod
+    async def need_migrate(self) -> bool:
+        """判断当前环境是否需要运行此迁移
+        """
+        pass
+
+    @abc.abstractmethod
+    async def run(self):
+        """执行迁移
+        """
+        pass

pkg/config/migrations/m1_sensitive_word_migration.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import os
+import sys
+
+from .. import migration
+
+
+@migration.migration_class("sensitive-word-migration", 1)
+class SensitiveWordMigration(migration.Migration):
+    """敏感词迁移
+    """
+
+    async def need_migrate(self) -> bool:
+        """判断当前环境是否需要运行此迁移
+        """
+        return os.path.exists("data/config/sensitive-words.json")
+
+    async def run(self):
+        """执行迁移
+        """
+        # 移动文件
+        os.rename("data/config/sensitive-words.json", "data/metadata/sensitive-words.json")
+
+        # 重新加载配置
+        await self.ap.sensitive_meta.load_config()

pkg/core/app.py

@@ -33,6 +33,8 @@ class Application:
 
     tool_mgr: llm_tool_mgr.ToolManager = None
 
+    # ======= 配置管理器 =======
+
     command_cfg: config_mgr.ConfigManager = None
 
     pipeline_cfg: config_mgr.ConfigManager = None
@@ -43,6 +45,16 @@ class Application:
 
     system_cfg: config_mgr.ConfigManager = None
 
+    # ======= 元数据配置管理器 =======
+
+    sensitive_meta: config_mgr.ConfigManager = None
+
+    adapter_qq_botpy_meta: config_mgr.ConfigManager = None
+
+    plugin_setting_meta: config_mgr.ConfigManager = None
+
+    # =========================
+
     ctr_mgr: center_mgr.V2CenterAPI = None
 
     plugin_mgr: plugin_mgr.PluginManager = None

pkg/core/boot.py

@@ -5,11 +5,12 @@ from ..audit import identifier
 from . import stage
 
 # 引入启动阶段实现以便注册
-from .stages import load_config, setup_logger, build_app
+from .stages import load_config, setup_logger, build_app, migrate
 
 
 stage_order = [
     "LoadConfigStage",
+    "MigrationStage",
     "SetupLoggerStage",
     "BuildAppStage"
 ]

pkg/core/bootutils/files.py

@@ -13,7 +13,6 @@ required_files = {
     "data/config/platform.json": "templates/platform.json",
     "data/config/provider.json": "templates/provider.json",
     "data/config/system.json": "templates/system.json",
-    "data/config/sensitive-words.json": "templates/sensitive-words.json",
     "data/scenario/default.json": "templates/scenario-template.json",
 }
 

pkg/core/stages/load_config.py

@@ -17,3 +17,13 @@ class LoadConfigStage(stage.BootingStage):
         ap.platform_cfg = await config.load_json_config("data/config/platform.json", "templates/platform.json")
         ap.provider_cfg = await config.load_json_config("data/config/provider.json", "templates/provider.json")
         ap.system_cfg = await config.load_json_config("data/config/system.json", "templates/system.json")
+
+
+        ap.plugin_setting_meta = await config.load_json_config("plugins/plugins.json", "templates/plugin-settings.json")
+        await ap.plugin_setting_meta.dump_config()
+
+        ap.sensitive_meta = await config.load_json_config("data/metadata/sensitive-words.json", "templates/metadata/sensitive-words.json")
+        await ap.sensitive_meta.dump_config()
+
+        ap.adapter_qq_botpy_meta = await config.load_json_config("data/metadata/adapter-qq-botpy.json", "templates/metadata/adapter-qq-botpy.json")
+        await ap.adapter_qq_botpy_meta.dump_config()

pkg/core/stages/migrate.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import importlib
+
+from .. import stage, app
+from ...config import migration
+from ...config.migrations import m1_sensitive_word_migration
+
+
+@stage.stage_class("MigrationStage")
+class MigrationStage(stage.BootingStage):
+    """迁移阶段
+    """
+
+    async def run(self, ap: app.Application):
+        """启动
+        """
+
+        migrations = migration.preregistered_migrations
+
+        # 按照迁移号排序
+        migrations.sort(key=lambda x: x.number)
+
+        for migration_cls in migrations:
+            migration_instance = migration_cls(ap)
+
+            if await migration_instance.need_migrate():
+                await migration_instance.run()

pkg/pipeline/cntfilter/filters/banwords.py

@@ -10,31 +10,26 @@ from ....config import manager as cfg_mgr
 class BanWordFilter(filter_model.ContentFilter):
     """根据内容禁言"""
 
-    sensitive: cfg_mgr.ConfigManager
-
     async def initialize(self):
-        self.sensitive = await cfg_mgr.load_json_config(
+        pass
-            "data/config/sensitive-words.json",
-            "templates/sensitive-words.json"
-        )
 
     async def process(self, message: str) -> entities.FilterResult:
         found = False
 
-        for word in self.sensitive.data['words']:
+        for word in self.ap.sensitive_meta.data['words']:
             match = re.findall(word, message)
 
             if len(match) > 0:
                 found = True
 
                 for i in range(len(match)):
-                    if self.sensitive.data['mask_word'] == "":
+                    if self.ap.sensitive_meta.data['mask_word'] == "":
                         message = message.replace(
-                            match[i], self.sensitive.data['mask'] * len(match[i])
+                            match[i], self.ap.sensitive_meta.data['mask'] * len(match[i])
                         )
                     else:
                         message = message.replace(
-                            match[i], self.sensitive.data['mask_word']
+                            match[i], self.ap.sensitive_meta.data['mask_word']
                         )
 
         return entities.FilterResult(

pkg/platform/sources/qqbotpy.py

@@ -506,10 +506,7 @@ class OfficialAdapter(adapter_model.MessageSourceAdapter):
 
     async def run_async(self):
 
-        self.metadata = await cfg_mgr.load_json_config(
+        self.metadata = self.ap.adapter_qq_botpy_meta
-            "data/metadata/adapter-qq-botpy.json",
-            "templates/metadata/adapter-qq-botpy.json",
-        )
 
         self.member_openid_mapping = OpenIDMapping(
             map=self.metadata.data["mapping"]["members"],

pkg/plugin/setting.py

@@ -16,10 +16,7 @@ class SettingManager:
         self.ap = ap
 
     async def initialize(self):
-        self.settings = await cfg_mgr.load_json_config(
+        self.settings = self.ap.plugin_setting_meta
-            'plugins/plugins.json',
-            'templates/plugin-settings.json'
-        )
 
     async def sync_setting(
         self,

templates/metadata/sensitive-words.json

@@ -0,0 +1,78 @@
+{
+    "说明": "mask将替换敏感词中的每一个字，若mask_word值不为空，则将敏感词整个替换为mask_word的值",
+    "mask": "*",
+    "mask_word": "",
+    "words": [
+        "习近平",
+        "胡锦涛",
+        "江泽民",
+        "温家宝",
+        "李克强",
+        "李长春",
+        "毛泽东",
+        "邓小平",
+        "周恩来",
+        "马克思",
+        "社会主义",
+        "共产党",
+        "共产主义",
+        "大陆官方",
+        "北京政权",
+        "中华帝国",
+        "中国政府",
+        "共狗",
+        "六四事件",
+        "天安门",
+        "六四",
+        "政治局常委",
+        "两会",
+        "共青团",
+        "学潮",
+        "八九",
+        "二十大",
+        "民进党",
+        "台独",
+        "台湾独立",
+        "台湾国",
+        "国民党",
+        "台湾民国",
+        "中华民国",
+        "pornhub",
+        "Pornhub",
+        "[Yy]ou[Pp]orn",
+        "porn",
+        "Porn",
+        "[Xx][Vv]ideos",
+        "[Rr]ed[Tt]ube",
+        "[Xx][Hh]amster",
+        "[Ss]pank[Ww]ire",
+        "[Ss]pank[Bb]ang",
+        "[Tt]ube8",
+        "[Yy]ou[Jj]izz",
+        "[Bb]razzers",
+        "[Nn]aughty[ ]?[Aa]merica",
+        "作爱",
+        "做爱",
+        "性交",
+        "性爱",
+        "自慰",
+        "阴茎",
+        "淫妇",
+        "肛交",
+        "交配",
+        "性关系",
+        "性活动",
+        "色情",
+        "色图",
+        "涩图",
+        "裸体",
+        "小穴",
+        "淫荡",
+        "性爱",
+        "翻墙",
+        "VPN",
+        "科学上网",
+        "挂梯子",
+        "GFW"
+    ]
+}

templates/sensitive-words.json

@@ -1,78 +0,0 @@
-{
-  "说明": "mask将替换敏感词中的每一个字，若mask_word值不为空，则将敏感词整个替换为mask_word的值",
-  "mask": "*",
-  "mask_word": "",
-  "words": [
-    "习近平",
-    "胡锦涛",
-    "江泽民",
-    "温家宝",
-    "李克强",
-    "李长春",
-    "毛泽东",
-    "邓小平",
-    "周恩来",
-    "马克思",
-    "社会主义",
-    "共产党",
-    "共产主义",
-    "大陆官方",
-    "北京政权",
-    "中华帝国",
-    "中国政府",
-    "共狗",
-    "六四事件",
-    "天安门",
-    "六四",
-    "政治局常委",
-    "两会",
-    "共青团",
-    "学潮",
-    "八九",
-    "二十大",
-    "民进党",
-    "台独",
-    "台湾独立",
-    "台湾国",
-    "国民党",
-    "台湾民国",
-    "中华民国",
-    "pornhub",
-    "Pornhub",
-    "[Yy]ou[Pp]orn",
-    "porn",
-    "Porn",
-    "[Xx][Vv]ideos",
-    "[Rr]ed[Tt]ube",
-    "[Xx][Hh]amster",
-    "[Ss]pank[Ww]ire",
-    "[Ss]pank[Bb]ang",
-    "[Tt]ube8",
-    "[Yy]ou[Jj]izz",
-    "[Bb]razzers",
-    "[Nn]aughty[ ]?[Aa]merica",
-    "作爱",
-    "做爱",
-    "性交",
-    "性爱",
-    "自慰",
-    "阴茎",
-    "淫妇",
-    "肛交",
-    "交配",
-    "性关系",
-    "性活动",
-    "色情",
-    "色图",
-    "涩图",
-    "裸体",
-    "小穴",
-    "淫荡",
-    "性爱",
-    "翻墙",
-    "VPN",
-    "科学上网",
-    "挂梯子",
-    "GFW"
-  ]
-}