diff --git a/pkg/api/http/controller/groups/settings.py b/pkg/api/http/controller/groups/settings.py deleted file mode 100644 index 835d86ad..00000000 --- a/pkg/api/http/controller/groups/settings.py +++ /dev/null @@ -1,62 +0,0 @@ -import quart - -from .....core import app -from .. import group - - -@group.group_class('settings', '/api/v1/settings') -class SettingsRouterGroup(group.RouterGroup): - - async def initialize(self) -> None: - - @self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN) - async def _() -> str: - return self.success( - data={ - "managers": [ - { - "name": m.name, - "description": m.description, - } - for m in self.ap.settings_mgr.get_manager_list() - ] - } - ) - - @self.route('/', methods=['GET'], auth_type=group.AuthType.USER_TOKEN) - async def _(manager_name: str) -> str: - - manager = self.ap.settings_mgr.get_manager(manager_name) - - if manager is None: - return self.fail(1, '配置管理器不存在') - - return self.success( - data={ - "manager": { - "name": manager.name, - "description": manager.description, - "schema": manager.schema, - "file": manager.file.config_file_name, - "data": manager.data, - "doc_link": manager.doc_link - } - } - ) - - @self.route('//data', methods=['PUT'], auth_type=group.AuthType.USER_TOKEN) - async def _(manager_name: str) -> str: - data = await quart.request.json - manager = self.ap.settings_mgr.get_manager(manager_name) - - if manager is None: - return self.fail(code=1, msg='配置管理器不存在') - - # manager.data = data['data'] - for k, v in data['data'].items(): - manager.data[k] = v - - await manager.dump_config() - return self.success(data={ - "data": manager.data - }) diff --git a/pkg/api/http/controller/main.py b/pkg/api/http/controller/main.py index 7e973b98..79838d7a 100644 --- a/pkg/api/http/controller/main.py +++ b/pkg/api/http/controller/main.py @@ -8,7 +8,7 @@ import quart_cors from ....core import app, entities as core_entities -from .groups import logs, system, settings, plugins, stats, user, pipelines +from .groups import logs, system, plugins, stats, user, pipelines from .groups.provider import models, requesters from .groups.platform import bots, adapters from . import group @@ -29,7 +29,7 @@ class HTTPController: await self.register_routes() async def run(self) -> None: - if self.ap.system_cfg.data["http-api"]["enable"]: + if True: async def shutdown_trigger_placeholder(): while True: @@ -45,8 +45,8 @@ class HTTPController: self.ap.task_mgr.create_task( exception_handler( - host=self.ap.system_cfg.data["http-api"]["host"], - port=self.ap.system_cfg.data["http-api"]["port"], + host='0.0.0.0', + port=self.ap.instance_config.data['api']['port'], shutdown_trigger=shutdown_trigger_placeholder, ), name="http-api-quart", diff --git a/pkg/api/http/service/user.py b/pkg/api/http/service/user.py index b09eef3c..4b31f1c0 100644 --- a/pkg/api/http/service/user.py +++ b/pkg/api/http/service/user.py @@ -56,8 +56,8 @@ class UserService: return await self.generate_jwt_token(user_email) async def generate_jwt_token(self, user_email: str) -> str: - jwt_secret = self.ap.instance_secret_meta.data['jwt_secret'] - jwt_expire = self.ap.system_cfg.data['http-api']['jwt-expire'] + jwt_secret = self.ap.instance_config.data['system']['jwt']['secret'] + jwt_expire = self.ap.instance_config.data['system']['jwt']['expire'] payload = { 'user': user_email, @@ -68,6 +68,6 @@ class UserService: return jwt.encode(payload, jwt_secret, algorithm='HS256') async def verify_jwt_token(self, token: str) -> str: - jwt_secret = self.ap.instance_secret_meta.data['jwt_secret'] + jwt_secret = self.ap.instance_config.data['system']['jwt']['secret'] return jwt.decode(token, jwt_secret, algorithms=['HS256'])['user'] diff --git a/pkg/audit/center/groups/main.py b/pkg/audit/center/groups/main.py index 3a31a65b..854437a1 100644 --- a/pkg/audit/center/groups/main.py +++ b/pkg/audit/center/groups/main.py @@ -12,7 +12,7 @@ class V2MainDataAPI(apigroup.APIGroup): super().__init__(prefix+"/main", ap) async def do(self, *args, **kwargs): - if not self.ap.system_cfg.data['report-usage']: + if not self.ap.instance_config.data['telemetry']['report']: return None return await super().do(*args, **kwargs) diff --git a/pkg/audit/center/groups/plugin.py b/pkg/audit/center/groups/plugin.py index 627b116c..d6ed0b02 100644 --- a/pkg/audit/center/groups/plugin.py +++ b/pkg/audit/center/groups/plugin.py @@ -12,7 +12,7 @@ class V2PluginDataAPI(apigroup.APIGroup): super().__init__(prefix+"/plugin", ap) async def do(self, *args, **kwargs): - if not self.ap.system_cfg.data['report-usage']: + if not self.ap.instance_config.data['telemetry']['report']: return None return await super().do(*args, **kwargs) diff --git a/pkg/audit/center/groups/usage.py b/pkg/audit/center/groups/usage.py index 8a8bdf04..79bc56f5 100644 --- a/pkg/audit/center/groups/usage.py +++ b/pkg/audit/center/groups/usage.py @@ -12,7 +12,7 @@ class V2UsageDataAPI(apigroup.APIGroup): super().__init__(prefix+"/usage", ap) async def do(self, *args, **kwargs): - if not self.ap.system_cfg.data['report-usage']: + if not self.ap.instance_config.data['telemetry']['report']: return None return await super().do(*args, **kwargs) diff --git a/pkg/command/cmdmgr.py b/pkg/command/cmdmgr.py index ea4e1a9b..3275b6fc 100644 --- a/pkg/command/cmdmgr.py +++ b/pkg/command/cmdmgr.py @@ -40,8 +40,8 @@ class CommandManager: # 应用命令权限配置 for cls in operator.preregistered_operators: - if cls.path in self.ap.command_cfg.data['privilege']: - cls.lowest_privilege = self.ap.command_cfg.data['privilege'][cls.path] + if cls.path in self.ap.instance_config.data['command']['privilege']: + cls.lowest_privilege = self.ap.instance_config.data['command']['privilege'][cls.path] # 实例化所有类 self.cmd_list = [cls(self.ap) for cls in operator.preregistered_operators] @@ -108,7 +108,7 @@ class CommandManager: privilege = 1 - if f'{query.launcher_type.value}_{query.launcher_id}' in self.ap.system_cfg.data['admin-sessions']: + if f'{query.launcher_type.value}_{query.launcher_id}' in self.ap.instance_config.data['admins']: privilege = 2 ctx = entities.ExecuteContext( diff --git a/pkg/command/operators/help.py b/pkg/command/operators/help.py index 570e103c..d8b42137 100644 --- a/pkg/command/operators/help.py +++ b/pkg/command/operators/help.py @@ -16,7 +16,7 @@ class HelpOperator(operator.CommandOperator): self, context: entities.ExecuteContext ) -> typing.AsyncGenerator[entities.CommandReturn, None]: - help = self.ap.system_cfg.data['help-message'] + help = 'LangBot - 大语言模型原生即时通信机器人平台\n链接:https://langbot.app' help += '\n发送命令 !cmd 可查看命令列表' diff --git a/pkg/config/settings.py b/pkg/config/settings.py deleted file mode 100644 index 1f21e926..00000000 --- a/pkg/config/settings.py +++ /dev/null @@ -1,75 +0,0 @@ -from __future__ import annotations - -from . import manager as config_manager -from ..core import app - - -class SettingsManager: - """设置管理器 - 保存、管理多个配置文件管理器 - """ - - ap: app.Application - - managers: list[config_manager.ConfigManager] = [] - """配置文件管理器列表""" - - def __init__(self, ap: app.Application) -> None: - self.ap = ap - self.managers = [] - - async def initialize(self) -> None: - pass - - def register_manager( - self, - name: str, - description: str, - manager: config_manager.ConfigManager, - schema: dict=None, - doc_link: str=None, - ) -> None: - """注册配置管理器 - - Args: - name (str): 配置管理器名 - description (str): 配置管理器描述 - manager (ConfigManager): 配置管理器 - schema (dict): 配置文件 schema,符合 JSON Schema Draft 7 规范 - """ - - for m in self.managers: - if m.name == name: - raise ValueError(f'配置管理器名 {name} 已存在') - - manager.name = name - manager.description = description - manager.schema = schema - manager.doc_link = doc_link - self.managers.append(manager) - - def get_manager(self, name: str) -> config_manager.ConfigManager | None: - """获取配置管理器 - - Args: - name (str): 配置管理器名 - - Returns: - ConfigManager: 配置管理器 - """ - - for m in self.managers: - if m.name == name: - return m - - return None - - def get_manager_list(self) -> list[config_manager.ConfigManager]: - """获取配置管理器列表 - - Returns: - list[ConfigManager]: 配置管理器列表 - """ - - return self.managers - diff --git a/pkg/core/app.py b/pkg/core/app.py index 43d124a4..e8409741 100644 --- a/pkg/core/app.py +++ b/pkg/core/app.py @@ -8,12 +8,11 @@ import enum import sys import os -from ..platform import manager as im_mgr +from ..platform import botmgr as im_mgr from ..provider.session import sessionmgr as llm_session_mgr from ..provider.modelmgr import modelmgr as llm_model_mgr from ..provider.tools import toolmgr as llm_tool_mgr from ..config import manager as config_mgr -from ..config import settings as settings_mgr from ..audit.center import v2 as center_mgr from ..command import cmdmgr from ..plugin import manager as plugin_mgr @@ -54,26 +53,24 @@ class Application: # TODO 移动到 pipeline 里 tool_mgr: llm_tool_mgr.ToolManager = None - settings_mgr: settings_mgr.SettingsManager = None - # ======= 配置管理器 ======= - command_cfg: config_mgr.ConfigManager = None + command_cfg: config_mgr.ConfigManager = None # deprecated - pipeline_cfg: config_mgr.ConfigManager = None + pipeline_cfg: config_mgr.ConfigManager = None # deprecated - platform_cfg: config_mgr.ConfigManager = None + platform_cfg: config_mgr.ConfigManager = None # deprecated - provider_cfg: config_mgr.ConfigManager = None + provider_cfg: config_mgr.ConfigManager = None # deprecated - system_cfg: config_mgr.ConfigManager = None + system_cfg: config_mgr.ConfigManager = None # deprecated + + instance_config: config_mgr.ConfigManager = None # ======= 元数据配置管理器 ======= sensitive_meta: config_mgr.ConfigManager = None - llm_models_meta: config_mgr.ConfigManager = None - instance_secret_meta: config_mgr.ConfigManager = None pipeline_config_meta_trigger: config_mgr.ConfigManager = None @@ -156,7 +153,7 @@ class Application: public_ip = await ip.get_myip() - port = self.system_cfg.data['http-api']['port'] + port = self.instance_config.data['api']['port'] tips = f""" ======================================= diff --git a/pkg/core/boot.py b/pkg/core/boot.py index e6a0e3eb..307fa95c 100644 --- a/pkg/core/boot.py +++ b/pkg/core/boot.py @@ -10,12 +10,13 @@ from . import stage from ..utils import constants # 引入启动阶段实现以便注册 -from .stages import load_config, setup_logger, build_app, migrate, show_notes +from .stages import load_config, setup_logger, build_app, migrate, show_notes, genkeys stage_order = [ "LoadConfigStage", "MigrationStage", + "GenKeysStage", "SetupLoggerStage", "BuildAppStage", "ShowNotesStage" diff --git a/pkg/core/bootutils/files.py b/pkg/core/bootutils/files.py index dbaad8ea..9a2dff71 100644 --- a/pkg/core/bootutils/files.py +++ b/pkg/core/bootutils/files.py @@ -7,11 +7,7 @@ import sys required_files = { "plugins/__init__.py": "templates/__init__.py", - "data/config/command.json": "templates/command.json", - "data/config/pipeline.json": "templates/pipeline.json", - "data/config/platform.json": "templates/platform.json", - "data/config/provider.json": "templates/provider.json", - "data/config/system.json": "templates/system.json", + "data/config.yaml": "templates/config.yaml", } required_paths = [ @@ -19,7 +15,6 @@ required_paths = [ "data", "data/metadata", "data/logs", - "data/config", "data/labels", "plugins" ] diff --git a/pkg/core/stages/build_app.py b/pkg/core/stages/build_app.py index fc049d9c..7d62e9c9 100644 --- a/pkg/core/stages/build_app.py +++ b/pkg/core/stages/build_app.py @@ -12,7 +12,7 @@ from ...command import cmdmgr from ...provider.session import sessionmgr as llm_session_mgr from ...provider.modelmgr import modelmgr as llm_model_mgr from ...provider.tools import toolmgr as llm_tool_mgr -from ...platform import manager as im_mgr +from ...platform import botmgr as im_mgr from ...persistence import mgr as persistencemgr from ...api.http.controller import main as http_controller from ...api.http.service import user as user_service @@ -50,7 +50,7 @@ class BuildAppStage(stage.BootingStage): center_v2_api = center_v2.V2CenterAPI( ap, - backend_url=ap.system_cfg.data["qcg-center-url"], + backend_url=ap.instance_config.data['telemetry']['url'], basic_info={ "host_id": identifier.identifier["host_id"], "instance_id": identifier.identifier["instance_id"], @@ -58,7 +58,7 @@ class BuildAppStage(stage.BootingStage): "platform": platform.get_platform(), }, runtime_info={ - "admin_id": "{}".format(ap.system_cfg.data["admin-sessions"]), + "admin_id": "{}".format(ap.instance_config.data["admins"]), "msg_source": str([]), }, ) diff --git a/pkg/core/stages/genkeys.py b/pkg/core/stages/genkeys.py new file mode 100644 index 00000000..843f1532 --- /dev/null +++ b/pkg/core/stages/genkeys.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import secrets +import os + +from .. import stage, app + + +@stage.stage_class("GenKeysStage") +class GenKeysStage(stage.BootingStage): + """生成密钥阶段 + """ + + async def run(self, ap: app.Application): + """启动 + """ + + if not ap.instance_config.data['system']['jwt']['secret']: + ap.instance_config.data['system']['jwt']['secret'] = secrets.token_hex(16) + await ap.instance_config.dump_config() diff --git a/pkg/core/stages/load_config.py b/pkg/core/stages/load_config.py index 97919901..edfe5915 100644 --- a/pkg/core/stages/load_config.py +++ b/pkg/core/stages/load_config.py @@ -5,7 +5,6 @@ import os from .. import stage, app from ..bootutils import config -from ...config import settings as settings_mgr @stage.stage_class("LoadConfigStage") @@ -16,33 +15,36 @@ class LoadConfigStage(stage.BootingStage): async def run(self, ap: app.Application): """启动 """ - - ap.settings_mgr = settings_mgr.SettingsManager(ap) - await ap.settings_mgr.initialize() + # ======= deprecated ======= if os.path.exists("data/config/command.json"): - ap.command_cfg = await config.load_json_config("data/config/command.json", "templates/command.json", completion=False) + ap.command_cfg = await config.load_json_config("data/config/command.json", "templates/legacy/command.json", completion=False) if os.path.exists("data/config/pipeline.json"): - ap.pipeline_cfg = await config.load_json_config("data/config/pipeline.json", "templates/pipeline.json", completion=False) + ap.pipeline_cfg = await config.load_json_config("data/config/pipeline.json", "templates/legacy/pipeline.json", completion=False) if os.path.exists("data/config/platform.json"): - ap.platform_cfg = await config.load_json_config("data/config/platform.json", "templates/platform.json", completion=False) + ap.platform_cfg = await config.load_json_config("data/config/platform.json", "templates/legacy/platform.json", completion=False) if os.path.exists("data/config/provider.json"): - ap.provider_cfg = await config.load_json_config("data/config/provider.json", "templates/provider.json", completion=False) + ap.provider_cfg = await config.load_json_config("data/config/provider.json", "templates/legacy/provider.json", completion=False) if os.path.exists("data/config/system.json"): - ap.system_cfg = await config.load_json_config("data/config/system.json", "templates/system.json", completion=False) + ap.system_cfg = await config.load_json_config("data/config/system.json", "templates/legacy/system.json", completion=False) + + if os.path.exists("data/metadata/instance-secret.json"): + ap.instance_secret_meta = await config.load_json_config("data/metadata/instance-secret.json", template_data={ + 'jwt_secret': secrets.token_hex(16) + }) + await ap.instance_secret_meta.dump_config() + # ======= deprecated ======= + + ap.instance_config = await config.load_yaml_config("data/config.yaml", "templates/config.yaml", completion=False) + await ap.instance_config.dump_config() ap.sensitive_meta = await config.load_json_config("data/metadata/sensitive-words.json", "templates/metadata/sensitive-words.json") await ap.sensitive_meta.dump_config() - ap.instance_secret_meta = await config.load_json_config("data/metadata/instance-secret.json", template_data={ - 'jwt_secret': secrets.token_hex(16) - }) - await ap.instance_secret_meta.dump_config() - ap.pipeline_config_meta_trigger = await config.load_yaml_config("templates/metadata/pipeline/trigger.yaml", "templates/metadata/pipeline/trigger.yaml") ap.pipeline_config_meta_safety = await config.load_yaml_config("templates/metadata/pipeline/safety.yaml", "templates/metadata/pipeline/safety.yaml") ap.pipeline_config_meta_ai = await config.load_yaml_config("templates/metadata/pipeline/ai.yaml", "templates/metadata/pipeline/ai.yaml") diff --git a/pkg/persistence/databases/sqlite.py b/pkg/persistence/databases/sqlite.py index 14f89092..5a815378 100644 --- a/pkg/persistence/databases/sqlite.py +++ b/pkg/persistence/databases/sqlite.py @@ -10,4 +10,4 @@ class SQLiteDatabaseManager(database.BaseDatabaseManager): """SQLite 数据库管理类""" async def initialize(self) -> None: - self.engine = sqlalchemy_asyncio.create_async_engine(f"sqlite+aiosqlite:///{self.ap.system_cfg.data['persistence']['sqlite']['path']}") + self.engine = sqlalchemy_asyncio.create_async_engine(f"sqlite+aiosqlite:///{self.ap.instance_config.data['persistence']['sqlite']['path']}") diff --git a/pkg/pipeline/controller.py b/pkg/pipeline/controller.py index 64d4e8f4..a7cf2153 100644 --- a/pkg/pipeline/controller.py +++ b/pkg/pipeline/controller.py @@ -20,7 +20,7 @@ class Controller: def __init__(self, ap: app.Application): self.ap = ap - self.semaphore = asyncio.Semaphore(self.ap.system_cfg.data['pipeline-concurrency']) + self.semaphore = asyncio.Semaphore(self.ap.instance_config.data['concurrency']['pipeline']) async def consumer(self): """事件处理循环 diff --git a/pkg/pipeline/process/handlers/command.py b/pkg/pipeline/process/handlers/command.py index cec64a45..b0316e1f 100644 --- a/pkg/pipeline/process/handlers/command.py +++ b/pkg/pipeline/process/handlers/command.py @@ -23,7 +23,7 @@ class CommandHandler(handler.MessageHandler): privilege = 1 - if f'{query.launcher_type.value}_{query.launcher_id}' in self.ap.system_cfg.data['admin-sessions']: + if f'{query.launcher_type.value}_{query.launcher_id}' in self.ap.instance_config.data['admins']: privilege = 2 spt = command_text.split(' ') diff --git a/pkg/pipeline/process/process.py b/pkg/pipeline/process/process.py index ea4d7e7f..11f43d3c 100644 --- a/pkg/pipeline/process/process.py +++ b/pkg/pipeline/process/process.py @@ -42,7 +42,7 @@ class Processor(stage.PipelineStage): self.ap.logger.info(f"处理 {query.launcher_type.value}_{query.launcher_id} 的请求({query.query_id}): {message_text}") async def generator(): - cmd_prefix = self.ap.command_cfg.data['command-prefix'] + cmd_prefix = self.ap.instance_config.data['command']['prefix'] if any(message_text.startswith(prefix) for prefix in cmd_prefix): async for result in self.cmd_handler.handle(query): diff --git a/pkg/platform/manager.py b/pkg/platform/botmgr.py similarity index 99% rename from pkg/platform/manager.py rename to pkg/platform/botmgr.py index 360f7588..8461461f 100644 --- a/pkg/platform/manager.py +++ b/pkg/platform/botmgr.py @@ -11,7 +11,7 @@ import sqlalchemy from .sources import qqofficial # FriendMessage, Image, MessageChain, Plain -from ..platform import adapter as msadapter +from . import adapter as msadapter from ..core import app, entities as core_entities, taskmgr from ..plugin import events diff --git a/pkg/provider/session/sessionmgr.py b/pkg/provider/session/sessionmgr.py index 5143f2bb..eef723da 100644 --- a/pkg/provider/session/sessionmgr.py +++ b/pkg/provider/session/sessionmgr.py @@ -29,10 +29,7 @@ class SessionManager: if query.launcher_type == session.launcher_type and query.launcher_id == session.launcher_id: return session - session_concurrency = self.ap.system_cfg.data['session-concurrency']['default'] - - if f'{query.launcher_type.value}_{query.launcher_id}' in self.ap.system_cfg.data['session-concurrency']: - session_concurrency = self.ap.system_cfg.data['session-concurrency'][f'{query.launcher_type.value}_{query.launcher_id}'] + session_concurrency = self.ap.instance_config.data['concurrency']['session'] session = core_entities.Session( launcher_type=query.launcher_type, diff --git a/pkg/provider/tools/loaders/mcp.py b/pkg/provider/tools/loaders/mcp.py index a475f9b7..6813ea37 100644 --- a/pkg/provider/tools/loaders/mcp.py +++ b/pkg/provider/tools/loaders/mcp.py @@ -108,7 +108,7 @@ class RuntimeMCPSession: """关闭工具""" await self.session._exit_stack.aclose() -@loader.loader_class("mcp") +# @loader.loader_class("mcp") class MCPLoader(loader.ToolLoader): """MCP 工具加载器。 diff --git a/pkg/utils/proxy.py b/pkg/utils/proxy.py index db03ad93..838814dd 100644 --- a/pkg/utils/proxy.py +++ b/pkg/utils/proxy.py @@ -25,10 +25,10 @@ class ProxyManager: "https://": os.getenv("HTTPS_PROXY") or os.getenv("https_proxy"), } - if 'http' in self.ap.system_cfg.data['network-proxies'] and self.ap.system_cfg.data['network-proxies']['http']: - self.forward_proxies['http://'] = self.ap.system_cfg.data['network-proxies']['http'] - if 'https' in self.ap.system_cfg.data['network-proxies'] and self.ap.system_cfg.data['network-proxies']['https']: - self.forward_proxies['https://'] = self.ap.system_cfg.data['network-proxies']['https'] + if 'http' in self.ap.instance_config.data['proxy'] and self.ap.instance_config.data['proxy']['http']: + self.forward_proxies['http://'] = self.ap.instance_config.data['proxy']['http'] + if 'https' in self.ap.instance_config.data['proxy'] and self.ap.instance_config.data['proxy']['https']: + self.forward_proxies['https://'] = self.ap.instance_config.data['proxy']['https'] # 设置到环境变量 os.environ['HTTP_PROXY'] = self.forward_proxies['http://'] or '' diff --git a/templates/config.yaml b/templates/config.yaml new file mode 100644 index 00000000..7f2d625e --- /dev/null +++ b/templates/config.yaml @@ -0,0 +1,24 @@ +admins: [] +api: + port: 5300 +command: + prefix: + - '!' + - ! + privilege: {} +concurrency: + pipeline: 20 + session: 1 +persistence: + sqlite: + path: data/langbot.db +proxy: + http: '' + https: '' +system: + jwt: + expire: 604800 + secret: bab3ba329e1fd7be096ac2c82f9b66f4 +telemetry: + report: true + url: https://api.qchatgpt.rockchin.top/api/v2 diff --git a/templates/command.json b/templates/legacy/command.json similarity index 100% rename from templates/command.json rename to templates/legacy/command.json diff --git a/templates/pipeline.json b/templates/legacy/pipeline.json similarity index 100% rename from templates/pipeline.json rename to templates/legacy/pipeline.json diff --git a/templates/platform.json b/templates/legacy/platform.json similarity index 100% rename from templates/platform.json rename to templates/legacy/platform.json diff --git a/templates/provider.json b/templates/legacy/provider.json similarity index 100% rename from templates/provider.json rename to templates/legacy/provider.json diff --git a/templates/system.json b/templates/legacy/system.json similarity index 100% rename from templates/system.json rename to templates/legacy/system.json