mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-06-05 05:16:03 +00:00
Merge pull request #729 from RockChinQ/feat/migration-stage
Feat: 配置文件迁移功能
This commit is contained in:
Junyan Qin
46
pkg/config/migration.py
Normal file
46
pkg/config/migration.py
Normal file
@@ -0,0 +1,46 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import typing
|
||||
|
||||
from ..core import app
|
||||
|
||||
|
||||
preregistered_migrations: list[typing.Type[Migration]] = []
|
||||
|
||||
def migration_class(name: str, number: int):
|
||||
"""注册一个迁移
|
||||
"""
|
||||
def decorator(cls: typing.Type[Migration]) -> typing.Type[Migration]:
|
||||
cls.name = name
|
||||
cls.number = number
|
||||
preregistered_migrations.append(cls)
|
||||
return cls
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
class Migration(abc.ABC):
|
||||
"""一个版本的迁移
|
||||
"""
|
||||
|
||||
name: str
|
||||
|
||||
number: int
|
||||
|
||||
ap: app.Application
|
||||
|
||||
def __init__(self, ap: app.Application):
|
||||
self.ap = ap
|
||||
|
||||
@abc.abstractmethod
|
||||
async def need_migrate(self) -> bool:
|
||||
"""判断当前环境是否需要运行此迁移
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def run(self):
|
||||
"""执行迁移
|
||||
"""
|
||||
pass
|
||||
0
pkg/config/migrations/__init__.py
Normal file
0
pkg/config/migrations/__init__.py
Normal file
26
pkg/config/migrations/m1_sensitive_word_migration.py
Normal file
26
pkg/config/migrations/m1_sensitive_word_migration.py
Normal file
@@ -0,0 +1,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class("sensitive-word-migration", 1)
|
||||
class SensitiveWordMigration(migration.Migration):
|
||||
"""敏感词迁移
|
||||
"""
|
||||
|
||||
async def need_migrate(self) -> bool:
|
||||
"""判断当前环境是否需要运行此迁移
|
||||
"""
|
||||
return os.path.exists("data/config/sensitive-words.json")
|
||||
|
||||
async def run(self):
|
||||
"""执行迁移
|
||||
"""
|
||||
# 移动文件
|
||||
os.rename("data/config/sensitive-words.json", "data/metadata/sensitive-words.json")
|
||||
|
||||
# 重新加载配置
|
||||
await self.ap.sensitive_meta.load_config()
|
||||
@@ -33,6 +33,8 @@ class Application:
|
||||
|
||||
tool_mgr: llm_tool_mgr.ToolManager = None
|
||||
|
||||
# ======= 配置管理器 =======
|
||||
|
||||
command_cfg: config_mgr.ConfigManager = None
|
||||
|
||||
pipeline_cfg: config_mgr.ConfigManager = None
|
||||
@@ -43,6 +45,16 @@ class Application:
|
||||
|
||||
system_cfg: config_mgr.ConfigManager = None
|
||||
|
||||
# ======= 元数据配置管理器 =======
|
||||
|
||||
sensitive_meta: config_mgr.ConfigManager = None
|
||||
|
||||
adapter_qq_botpy_meta: config_mgr.ConfigManager = None
|
||||
|
||||
plugin_setting_meta: config_mgr.ConfigManager = None
|
||||
|
||||
# =========================
|
||||
|
||||
ctr_mgr: center_mgr.V2CenterAPI = None
|
||||
|
||||
plugin_mgr: plugin_mgr.PluginManager = None
|
||||
|
||||
@@ -5,11 +5,12 @@ from ..audit import identifier
|
||||
from . import stage
|
||||
|
||||
# 引入启动阶段实现以便注册
|
||||
from .stages import load_config, setup_logger, build_app
|
||||
from .stages import load_config, setup_logger, build_app, migrate
|
||||
|
||||
|
||||
stage_order = [
|
||||
"LoadConfigStage",
|
||||
"MigrationStage",
|
||||
"SetupLoggerStage",
|
||||
"BuildAppStage"
|
||||
]
|
||||
|
||||
@@ -13,7 +13,6 @@ required_files = {
|
||||
"data/config/platform.json": "templates/platform.json",
|
||||
"data/config/provider.json": "templates/provider.json",
|
||||
"data/config/system.json": "templates/system.json",
|
||||
"data/config/sensitive-words.json": "templates/sensitive-words.json",
|
||||
"data/scenario/default.json": "templates/scenario-template.json",
|
||||
}
|
||||
|
||||
|
||||
@@ -17,3 +17,13 @@ class LoadConfigStage(stage.BootingStage):
|
||||
ap.platform_cfg = await config.load_json_config("data/config/platform.json", "templates/platform.json")
|
||||
ap.provider_cfg = await config.load_json_config("data/config/provider.json", "templates/provider.json")
|
||||
ap.system_cfg = await config.load_json_config("data/config/system.json", "templates/system.json")
|
||||
|
||||
|
||||
ap.plugin_setting_meta = await config.load_json_config("plugins/plugins.json", "templates/plugin-settings.json")
|
||||
await ap.plugin_setting_meta.dump_config()
|
||||
|
||||
ap.sensitive_meta = await config.load_json_config("data/metadata/sensitive-words.json", "templates/metadata/sensitive-words.json")
|
||||
await ap.sensitive_meta.dump_config()
|
||||
|
||||
ap.adapter_qq_botpy_meta = await config.load_json_config("data/metadata/adapter-qq-botpy.json", "templates/metadata/adapter-qq-botpy.json")
|
||||
await ap.adapter_qq_botpy_meta.dump_config()
|
||||
|
||||
28
pkg/core/stages/migrate.py
Normal file
28
pkg/core/stages/migrate.py
Normal file
@@ -0,0 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
|
||||
from .. import stage, app
|
||||
from ...config import migration
|
||||
from ...config.migrations import m1_sensitive_word_migration
|
||||
|
||||
|
||||
@stage.stage_class("MigrationStage")
|
||||
class MigrationStage(stage.BootingStage):
|
||||
"""迁移阶段
|
||||
"""
|
||||
|
||||
async def run(self, ap: app.Application):
|
||||
"""启动
|
||||
"""
|
||||
|
||||
migrations = migration.preregistered_migrations
|
||||
|
||||
# 按照迁移号排序
|
||||
migrations.sort(key=lambda x: x.number)
|
||||
|
||||
for migration_cls in migrations:
|
||||
migration_instance = migration_cls(ap)
|
||||
|
||||
if await migration_instance.need_migrate():
|
||||
await migration_instance.run()
|
||||
@@ -10,31 +10,26 @@ from ....config import manager as cfg_mgr
|
||||
class BanWordFilter(filter_model.ContentFilter):
|
||||
"""根据内容禁言"""
|
||||
|
||||
sensitive: cfg_mgr.ConfigManager
|
||||
|
||||
async def initialize(self):
|
||||
self.sensitive = await cfg_mgr.load_json_config(
|
||||
"data/config/sensitive-words.json",
|
||||
"templates/sensitive-words.json"
|
||||
)
|
||||
pass
|
||||
|
||||
async def process(self, message: str) -> entities.FilterResult:
|
||||
found = False
|
||||
|
||||
for word in self.sensitive.data['words']:
|
||||
for word in self.ap.sensitive_meta.data['words']:
|
||||
match = re.findall(word, message)
|
||||
|
||||
if len(match) > 0:
|
||||
found = True
|
||||
|
||||
for i in range(len(match)):
|
||||
if self.sensitive.data['mask_word'] == "":
|
||||
if self.ap.sensitive_meta.data['mask_word'] == "":
|
||||
message = message.replace(
|
||||
match[i], self.sensitive.data['mask'] * len(match[i])
|
||||
match[i], self.ap.sensitive_meta.data['mask'] * len(match[i])
|
||||
)
|
||||
else:
|
||||
message = message.replace(
|
||||
match[i], self.sensitive.data['mask_word']
|
||||
match[i], self.ap.sensitive_meta.data['mask_word']
|
||||
)
|
||||
|
||||
return entities.FilterResult(
|
||||
|
||||
@@ -506,10 +506,7 @@ class OfficialAdapter(adapter_model.MessageSourceAdapter):
|
||||
|
||||
async def run_async(self):
|
||||
|
||||
self.metadata = await cfg_mgr.load_json_config(
|
||||
"data/metadata/adapter-qq-botpy.json",
|
||||
"templates/metadata/adapter-qq-botpy.json",
|
||||
)
|
||||
self.metadata = self.ap.adapter_qq_botpy_meta
|
||||
|
||||
self.member_openid_mapping = OpenIDMapping(
|
||||
map=self.metadata.data["mapping"]["members"],
|
||||
|
||||
@@ -16,10 +16,7 @@ class SettingManager:
|
||||
self.ap = ap
|
||||
|
||||
async def initialize(self):
|
||||
self.settings = await cfg_mgr.load_json_config(
|
||||
'plugins/plugins.json',
|
||||
'templates/plugin-settings.json'
|
||||
)
|
||||
self.settings = self.ap.plugin_setting_meta
|
||||
|
||||
async def sync_setting(
|
||||
self,
|
||||
|
||||
78
templates/metadata/sensitive-words.json
Normal file
78
templates/metadata/sensitive-words.json
Normal file
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"说明": "mask将替换敏感词中的每一个字，若mask_word值不为空，则将敏感词整个替换为mask_word的值",
|
||||
"mask": "*",
|
||||
"mask_word": "",
|
||||
"words": [
|
||||
"习近平",
|
||||
"胡锦涛",
|
||||
"江泽民",
|
||||
"温家宝",
|
||||
"李克强",
|
||||
"李长春",
|
||||
"毛泽东",
|
||||
"邓小平",
|
||||
"周恩来",
|
||||
"马克思",
|
||||
"社会主义",
|
||||
"共产党",
|
||||
"共产主义",
|
||||
"大陆官方",
|
||||
"北京政权",
|
||||
"中华帝国",
|
||||
"中国政府",
|
||||
"共狗",
|
||||
"六四事件",
|
||||
"天安门",
|
||||
"六四",
|
||||
"政治局常委",
|
||||
"两会",
|
||||
"共青团",
|
||||
"学潮",
|
||||
"八九",
|
||||
"二十大",
|
||||
"民进党",
|
||||
"台独",
|
||||
"台湾独立",
|
||||
"台湾国",
|
||||
"国民党",
|
||||
"台湾民国",
|
||||
"中华民国",
|
||||
"pornhub",
|
||||
"Pornhub",
|
||||
"[Yy]ou[Pp]orn",
|
||||
"porn",
|
||||
"Porn",
|
||||
"[Xx][Vv]ideos",
|
||||
"[Rr]ed[Tt]ube",
|
||||
"[Xx][Hh]amster",
|
||||
"[Ss]pank[Ww]ire",
|
||||
"[Ss]pank[Bb]ang",
|
||||
"[Tt]ube8",
|
||||
"[Yy]ou[Jj]izz",
|
||||
"[Bb]razzers",
|
||||
"[Nn]aughty[ ]?[Aa]merica",
|
||||
"作爱",
|
||||
"做爱",
|
||||
"性交",
|
||||
"性爱",
|
||||
"自慰",
|
||||
"阴茎",
|
||||
"淫妇",
|
||||
"肛交",
|
||||
"交配",
|
||||
"性关系",
|
||||
"性活动",
|
||||
"色情",
|
||||
"色图",
|
||||
"涩图",
|
||||
"裸体",
|
||||
"小穴",
|
||||
"淫荡",
|
||||
"性爱",
|
||||
"翻墙",
|
||||
"VPN",
|
||||
"科学上网",
|
||||
"挂梯子",
|
||||
"GFW"
|
||||
]
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
{
|
||||
"说明": "mask将替换敏感词中的每一个字，若mask_word值不为空，则将敏感词整个替换为mask_word的值",
|
||||
"mask": "*",
|
||||
"mask_word": "",
|
||||
"words": [
|
||||
"习近平",
|
||||
"胡锦涛",
|
||||
"江泽民",
|
||||
"温家宝",
|
||||
"李克强",
|
||||
"李长春",
|
||||
"毛泽东",
|
||||
"邓小平",
|
||||
"周恩来",
|
||||
"马克思",
|
||||
"社会主义",
|
||||
"共产党",
|
||||
"共产主义",
|
||||
"大陆官方",
|
||||
"北京政权",
|
||||
"中华帝国",
|
||||
"中国政府",
|
||||
"共狗",
|
||||
"六四事件",
|
||||
"天安门",
|
||||
"六四",
|
||||
"政治局常委",
|
||||
"两会",
|
||||
"共青团",
|
||||
"学潮",
|
||||
"八九",
|
||||
"二十大",
|
||||
"民进党",
|
||||
"台独",
|
||||
"台湾独立",
|
||||
"台湾国",
|
||||
"国民党",
|
||||
"台湾民国",
|
||||
"中华民国",
|
||||
"pornhub",
|
||||
"Pornhub",
|
||||
"[Yy]ou[Pp]orn",
|
||||
"porn",
|
||||
"Porn",
|
||||
"[Xx][Vv]ideos",
|
||||
"[Rr]ed[Tt]ube",
|
||||
"[Xx][Hh]amster",
|
||||
"[Ss]pank[Ww]ire",
|
||||
"[Ss]pank[Bb]ang",
|
||||
"[Tt]ube8",
|
||||
"[Yy]ou[Jj]izz",
|
||||
"[Bb]razzers",
|
||||
"[Nn]aughty[ ]?[Aa]merica",
|
||||
"作爱",
|
||||
"做爱",
|
||||
"性交",
|
||||
"性爱",
|
||||
"自慰",
|
||||
"阴茎",
|
||||
"淫妇",
|
||||
"肛交",
|
||||
"交配",
|
||||
"性关系",
|
||||
"性活动",
|
||||
"色情",
|
||||
"色图",
|
||||
"涩图",
|
||||
"裸体",
|
||||
"小穴",
|
||||
"淫荡",
|
||||
"性爱",
|
||||
"翻墙",
|
||||
"VPN",
|
||||
"科学上网",
|
||||
"挂梯子",
|
||||
"GFW"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user