Merge branch 'RockChinQ:master' into master

pkg/command/operators/func.py

@@ -16,7 +16,6 @@ class FuncOperator(operator.CommandOperator):
 
         all_functions = await self.ap.tool_mgr.get_all_functions(
             plugin_enabled=True,
-            plugin_status=plugin_context.RuntimeContainerStatus.INITIALIZED,
         )
 
         for func in all_functions:

pkg/core/app.py

@@ -204,6 +204,8 @@ class Application:
             case core_entities.LifecycleControlScope.PROVIDER.value:
                 self.logger.info("执行热重载 scope="+scope)
 
+                await self.tool_mgr.shutdown()
+
                 latest_llm_model_config = await config.load_json_config("data/metadata/llm-models.json", "templates/metadata/llm-models.json")
                 self.llm_models_meta = latest_llm_model_config
                 llm_model_mgr_inst = llm_model_mgr.ModelManager(self)

pkg/core/bootutils/deps.py

@@ -33,6 +33,8 @@ required_deps = {
     "dingtalk_stream": "dingtalk_stream",
     "dashscope": "dashscope",
     "telegram": "python-telegram-bot",
+    "certifi": "certifi",
+    "mcp": "mcp",
 }
 
 

pkg/core/migrations/m036_wxoa_loading_message.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from .. import migration
+
+
+@migration.migration_class("wxoa-loading-message", 36)
+class WxoaLoadingMessageMigration(migration.Migration):
+    """迁移"""
+
+    async def need_migrate(self) -> bool:
+        """判断当前环境是否需要运行此迁移"""
+
+        for adapter in self.ap.platform_cfg.data['platform-adapters']:
+            if adapter['adapter'] == 'officialaccount':
+                if 'LoadingMessage' not in adapter:
+                    return True
+        return False
+
+    async def run(self):
+        """执行迁移"""
+        for adapter in self.ap.platform_cfg.data['platform-adapters']:
+            if adapter['adapter'] == 'officialaccount':
+                if 'LoadingMessage' not in adapter:
+                    adapter['LoadingMessage'] = 'AI正在思考中，请发送任意内容获取回复。'
+
+        await self.ap.platform_cfg.dump_config()

pkg/core/migrations/m037_mcp_config.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from .. import migration
+
+
+@migration.migration_class("mcp-config", 37)
+class MCPConfigMigration(migration.Migration):
+    """迁移"""
+
+    async def need_migrate(self) -> bool:
+        """判断当前环境是否需要运行此迁移"""
+        return 'mcp' not in self.ap.provider_cfg.data
+
+    async def run(self):
+        """执行迁移"""
+        self.ap.provider_cfg.data['mcp'] = {
+            "servers": []
+        }
+
+        await self.ap.provider_cfg.dump_config()

pkg/core/stages/migrate.py

@@ -11,7 +11,9 @@ from ..migrations import m015_gitee_ai_config, m016_dify_service_api, m017_dify_
 from ..migrations import m020_wecom_config, m021_lark_config, m022_lmstudio_config, m023_siliconflow_config, m024_discord_config, m025_gewechat_config
 from ..migrations import m026_qqofficial_config, m027_wx_official_account_config, m028_aliyun_requester_config
 from ..migrations import m029_dashscope_app_api_config, m030_lark_config_cmpl, m031_dingtalk_config, m032_volcark_config
-from ..migrations import m033_dify_thinking_config, m034_gewechat_file_url_config, m035_wxoa_mode
+from ..migrations import m033_dify_thinking_config, m034_gewechat_file_url_config, m035_wxoa_mode, m036_wxoa_loading_message
+from ..migrations import m037_mcp_config
+
 
 @stage.stage_class("MigrationStage")
 class MigrationStage(stage.BootingStage):

pkg/pipeline/preproc/preproc.py

@@ -60,11 +60,14 @@ class PreProcessor(stage.PipelineStage):
 
         content_list = []
 
+        plain_text = ""
+
         for me in query.message_chain:
             if isinstance(me, platform_message.Plain):
                 content_list.append(
                     llm_entities.ContentElement.from_text(me.text)
                 )
+                plain_text += me.text
             elif isinstance(me, platform_message.Image):
                 if self.ap.provider_cfg.data['enable-vision'] and (self.ap.provider_cfg.data['runner'] != 'local-agent' or query.use_model.vision_supported):
                     if me.base64 is not None:
@@ -72,6 +75,8 @@ class PreProcessor(stage.PipelineStage):
                             llm_entities.ContentElement.from_image_base64(me.base64)
                         )
 
+        query.variables['user_message_text'] = plain_text
+
         query.user_message = llm_entities.Message(
             role='user',
             content=content_list

pkg/platform/sources/officialaccount.py

@@ -107,6 +107,7 @@ class OfficialAccountAdapter(adapter.MessagePlatformAdapter):
                 EncodingAESKey=config['EncodingAESKey'],
                 Appsecret=config['AppSecret'],
                 AppID=config['AppID'], 
+                LoadingMessage=config['LoadingMessage']
             )
         else:
             raise KeyError("请设置微信公众号通信模式")

pkg/plugin/installers/github.py

@@ -4,6 +4,8 @@ import re
 import os
 import shutil
 import zipfile
+import ssl
+import certifi
 
 import aiohttp
 import aiofiles
@@ -21,44 +23,39 @@ class GitHubRepoInstaller(installer.PluginInstaller):
 
     def get_github_plugin_repo_label(self, repo_url: str) -> list[str]:
         """获取username, repo"""
-
-        # 提取 username/repo , 正则表达式
         repo = re.findall(
             r"(?:https?://github\.com/|git@github\.com:)([^/]+/[^/]+?)(?:\.git|/|$)",
             repo_url,
         )
-
-        if len(repo) > 0:  # github
+        if len(repo) > 0:
             return repo[0].split("/")
         else:
             return None
-    
+
     async def download_plugin_source_code(self, repo_url: str, target_path: str, task_context: taskmgr.TaskContext = taskmgr.TaskContext.placeholder()) -> str:
         """下载插件源码（全异步）"""
-        
-        # 提取 username/repo , 正则表达式
         repo = self.get_github_plugin_repo_label(repo_url)
-
-        target_path += repo[1]
-
         if repo is None:
             raise errors.PluginInstallerError('仅支持GitHub仓库地址')
         
+        target_path += repo[1]
         self.ap.logger.debug("正在下载源码...")
         task_context.trace("下载源码...", "download-plugin-source-code")
         
         zipball_url = f"https://api.github.com/repos/{'/'.join(repo)}/zipball/HEAD"
-
         zip_resp: bytes = None
+        
+        # 创建自定义SSL上下文，使用certifi提供的根证书
+        ssl_context = ssl.create_default_context(cafile=certifi.where())
 
         async with aiohttp.ClientSession(trust_env=True) as session:
             async with session.get(
                 url=zipball_url,
-                timeout=aiohttp.ClientTimeout(total=300)
+                timeout=aiohttp.ClientTimeout(total=300),
+                ssl=ssl_context  # 使用自定义SSL上下文来验证证书
             ) as resp:
                 if resp.status != 200:
-                    raise errors.PluginInstallerError(f"下载源码失败: {resp.text}")
-                
+                    raise errors.PluginInstallerError(f"下载源码失败: {await resp.text()}")
                 zip_resp = await resp.read()
         
         if await aiofiles_os.path.exists("temp/" + target_path):
@@ -80,15 +77,11 @@ class GitHubRepoInstaller(installer.PluginInstaller):
         await aiofiles_os.remove("temp/" + target_path + "/source.zip")
 
         import glob
-
         unzip_dir = glob.glob("temp/" + target_path + "/*")[0]
-
         await aioshutil.copytree(unzip_dir, target_path + "/")
-
         await aioshutil.rmtree(unzip_dir)
         
         self.ap.logger.debug("源码下载完成。")
-
         return repo[1]
 
     async def install_requirements(self, path: str):
@@ -100,20 +93,14 @@ class GitHubRepoInstaller(installer.PluginInstaller):
         plugin_source: str,
         task_context: taskmgr.TaskContext = taskmgr.TaskContext.placeholder(),
     ):
-        """安装插件
-        """
+        """安装插件"""
         task_context.trace("下载插件源码...", "install-plugin")
-
         repo_label = await self.download_plugin_source_code(plugin_source, "plugins/", task_context)
-
         task_context.trace("安装插件依赖...", "install-plugin")
-
         await self.install_requirements("plugins/" + repo_label)
-
         task_context.trace("完成.", "install-plugin")
-
         await self.ap.plugin_mgr.setting.record_installed_plugin_source(
-            "plugins/"+repo_label+'/', plugin_source
+            "plugins/" + repo_label + '/', plugin_source
         )
 
     async def uninstall_plugin(
@@ -121,10 +108,8 @@ class GitHubRepoInstaller(installer.PluginInstaller):
         plugin_name: str,
         task_context: taskmgr.TaskContext = taskmgr.TaskContext.placeholder(),
     ):
-        """卸载插件
-        """
+        """卸载插件"""
         plugin_container = self.ap.plugin_mgr.get_plugin_by_name(plugin_name)
-
         if plugin_container is None:
             raise errors.PluginInstallerError('插件不存在或未成功加载')
         else:
@@ -135,24 +120,18 @@ class GitHubRepoInstaller(installer.PluginInstaller):
     async def update_plugin(
         self,
         plugin_name: str,
-        plugin_source: str=None,
+        plugin_source: str = None,
         task_context: taskmgr.TaskContext = taskmgr.TaskContext.placeholder(),
     ):
-        """更新插件
-        """
+        """更新插件"""
         task_context.trace("更新插件...", "update-plugin")
-
         plugin_container = self.ap.plugin_mgr.get_plugin_by_name(plugin_name)
-
         if plugin_container is None:
             raise errors.PluginInstallerError('插件不存在或未成功加载')
         else:
             if plugin_container.plugin_source:
                 plugin_source = plugin_container.plugin_source
-
                 task_context.trace("转交安装任务.", "update-plugin")
-
                 await self.install_plugin(plugin_source, task_context)
-
             else:
-                raise errors.PluginInstallerError('插件无源码信息，无法更新')
+                raise errors.PluginInstallerError('插件无源码信息，无法更新')

pkg/provider/session/sessionmgr.py

@@ -54,7 +54,6 @@ class SessionManager:
                 use_model=await self.ap.model_mgr.get_model_by_name(self.ap.provider_cfg.data['model']),
                 use_funcs=await self.ap.tool_mgr.get_all_functions(
                     plugin_enabled=True,
-                    plugin_status=plugin_context.RuntimeContainerStatus.INITIALIZED,
                 ),
             )
             session.conversations.append(conversation)

pkg/provider/tools/loader.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import abc
+import typing
+
+from ...core import app, entities as core_entities
+from . import entities as tools_entities
+
+
+preregistered_loaders: list[typing.Type[ToolLoader]] = []
+
+def loader_class(name: str):
+    """注册一个工具加载器
+    """
+    def decorator(cls: typing.Type[ToolLoader]) -> typing.Type[ToolLoader]:
+        cls.name = name
+        preregistered_loaders.append(cls)
+        return cls
+
+    return decorator
+
+
+class ToolLoader(abc.ABC):
+    """工具加载器"""
+    
+    name: str = None
+
+    ap: app.Application
+
+    def __init__(self, ap: app.Application):
+        self.ap = ap
+
+    async def initialize(self):
+        pass
+
+    @abc.abstractmethod
+    async def get_tools(self, enabled: bool=True) -> list[tools_entities.LLMFunction]:
+        """获取所有工具"""
+        pass
+
+    @abc.abstractmethod
+    async def has_tool(self, name: str) -> bool:
+        """检查工具是否存在"""
+        pass
+
+    @abc.abstractmethod
+    async def invoke_tool(self, query: core_entities.Query, name: str, parameters: dict) -> typing.Any:
+        """执行工具调用"""
+        pass
+
+    @abc.abstractmethod
+    async def shutdown(self):
+        """关闭工具"""
+        pass

pkg/provider/tools/loaders/mcp.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import typing
+from contextlib import AsyncExitStack
+
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+from mcp.client.sse import sse_client
+
+from .. import loader, entities as tools_entities
+from ....core import app, entities as core_entities
+
+
+class RuntimeMCPSession:
+    """运行时 MCP 会话"""
+
+    ap: app.Application
+
+    server_name: str
+
+    server_config: dict
+
+    session: ClientSession
+
+    exit_stack: AsyncExitStack
+
+    functions: list[tools_entities.LLMFunction] = []
+
+    def __init__(self, server_name: str, server_config: dict, ap: app.Application):
+        self.server_name = server_name
+        self.server_config = server_config
+        self.ap = ap
+        
+        self.session = None
+
+        self.exit_stack = AsyncExitStack()
+        self.functions = []
+
+    async def _init_stdio_python_server(self):
+        server_params = StdioServerParameters(
+            command=self.server_config["command"],
+            args=self.server_config["args"],
+            env=self.server_config["env"],
+        )
+
+        stdio_transport = await self.exit_stack.enter_async_context(
+            stdio_client(server_params)
+        )
+
+        stdio, write = stdio_transport
+
+        self.session = await self.exit_stack.enter_async_context(
+            ClientSession(stdio, write)
+        )
+
+        await self.session.initialize()
+
+    async def _init_sse_server(self):
+        sse_transport = await self.exit_stack.enter_async_context(
+            sse_client(
+                self.server_config["url"],
+                headers=self.server_config.get("headers", {}),
+                timeout=self.server_config.get("timeout", 10),
+            )
+        )
+        
+        sseio, write = sse_transport
+
+        self.session = await self.exit_stack.enter_async_context(
+            ClientSession(sseio, write)
+        )
+
+        await self.session.initialize()
+
+    async def initialize(self):
+        self.ap.logger.debug(f"初始化 MCP 会话: {self.server_name} {self.server_config}")
+
+        if self.server_config["mode"] == "stdio":
+            await self._init_stdio_python_server()
+        elif self.server_config["mode"] == "sse":
+            await self._init_sse_server()
+        else:
+            raise ValueError(f"无法识别 MCP 服务器类型: {self.server_name}: {self.server_config}")
+        
+        tools = await self.session.list_tools()
+
+        self.ap.logger.debug(f"获取 MCP 工具: {tools}")
+
+        for tool in tools.tools:
+
+            async def func(query: core_entities.Query, **kwargs):
+                result = await self.session.call_tool(tool.name, kwargs)
+                if result.isError:
+                    raise Exception(result.content[0].text)
+                return result.content[0].text
+            
+            func.__name__ = tool.name
+
+            self.functions.append(tools_entities.LLMFunction(
+                name=tool.name,
+                human_desc=tool.description,
+                description=tool.description,
+                parameters=tool.inputSchema,
+                func=func,
+            ))
+
+    async def shutdown(self):
+        """关闭工具"""
+        await self.session._exit_stack.aclose()
+
+@loader.loader_class("mcp")
+class MCPLoader(loader.ToolLoader):
+    """MCP 工具加载器。
+    
+    在此加载器中管理所有与 MCP Server 的连接。
+    """
+
+    sessions: dict[str, RuntimeMCPSession] = {}
+
+    _last_listed_functions: list[tools_entities.LLMFunction] = []
+
+    def __init__(self, ap: app.Application):
+        super().__init__(ap)
+        self.sessions = {}
+        self._last_listed_functions = []
+
+    async def initialize(self):
+
+        for server_config in self.ap.provider_cfg.data.get("mcp", {}).get("servers", []):
+            if not server_config["enable"]:
+                continue
+            session = RuntimeMCPSession(server_config["name"], server_config, self.ap)
+            await session.initialize()
+            # self.ap.event_loop.create_task(session.initialize())
+            self.sessions[server_config["name"]] = session
+
+    async def get_tools(self, enabled: bool=True) -> list[tools_entities.LLMFunction]:
+        all_functions = []
+
+        for session in self.sessions.values():
+            all_functions.extend(session.functions)
+
+        self._last_listed_functions = all_functions
+
+        return all_functions
+
+    async def has_tool(self, name: str) -> bool:
+        return name in [f.name for f in self._last_listed_functions]
+
+    async def invoke_tool(self, query: core_entities.Query, name: str, parameters: dict) -> typing.Any:
+        for server_name, session in self.sessions.items():
+            for function in session.functions:
+                if function.name == name:
+                    return await function.func(query, **parameters)
+
+        raise ValueError(f"未找到工具: {name}")
+
+    async def shutdown(self):
+        """关闭工具"""
+        for session in self.sessions.values():
+            await session.shutdown()

pkg/provider/tools/loaders/plugin.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import typing
+import traceback
+
+from .. import loader, entities as tools_entities
+from ....core import app, entities as core_entities
+from ....plugin import context as plugin_context
+
+
+@loader.loader_class("plugin-tool-loader")
+class PluginToolLoader(loader.ToolLoader):
+    """插件工具加载器。
+    
+    本加载器中不存储工具信息，仅负责从插件系统中获取工具信息。
+    """
+
+    async def get_tools(self, enabled: bool=True) -> list[tools_entities.LLMFunction]:
+
+        # 从插件系统获取工具（内容函数）
+        all_functions: list[tools_entities.LLMFunction] = []
+
+        for plugin in self.ap.plugin_mgr.plugins(
+            enabled=enabled, status=plugin_context.RuntimeContainerStatus.INITIALIZED
+        ):
+            all_functions.extend(plugin.content_functions)
+
+        return all_functions
+
+    async def has_tool(self, name: str) -> bool:
+        """检查工具是否存在"""
+        for plugin in self.ap.plugin_mgr.plugins(
+            enabled=True, status=plugin_context.RuntimeContainerStatus.INITIALIZED
+        ):
+            for function in plugin.content_functions:
+                if function.name == name:
+                    return True
+        return False
+
+    async def _get_function_and_plugin(
+        self, name: str
+    ) -> typing.Tuple[tools_entities.LLMFunction, plugin_context.BasePlugin]:
+        """获取函数和插件实例"""
+        for plugin in self.ap.plugin_mgr.plugins(
+            enabled=True, status=plugin_context.RuntimeContainerStatus.INITIALIZED
+        ):
+            for function in plugin.content_functions:
+                if function.name == name:
+                    return function, plugin.plugin_inst
+        return None, None
+
+    async def invoke_tool(self, query: core_entities.Query, name: str, parameters: dict) -> typing.Any:
+
+        try:
+
+            function, plugin = await self._get_function_and_plugin(name)
+            if function is None:
+                return None
+
+            parameters = parameters.copy()
+
+            parameters = {"query": query, **parameters}
+
+            return await function.func(plugin, **parameters)
+        except Exception as e:
+            self.ap.logger.error(f"执行函数 {name} 时发生错误: {e}")
+            traceback.print_exc()
+            return f"error occurred when executing function {name}: {e}"
+        finally:
+            plugin = None
+
+            for p in self.ap.plugin_mgr.plugins():
+                if function in p.content_functions:
+                    plugin = p
+                    break
+
+            if plugin is not None:
+
+                await self.ap.ctr_mgr.usage.post_function_record(
+                    plugin={
+                        "name": plugin.plugin_name,
+                        "remote": plugin.plugin_source,
+                        "version": plugin.plugin_version,
+                        "author": plugin.plugin_author,
+                    },
+                    function_name=function.name,
+                    function_description=function.description,
+                )
+
+    async def shutdown(self):
+        """关闭工具"""
+        pass

pkg/provider/tools/toolmgr.py

@@ -4,8 +4,9 @@ import typing
 import traceback
 
 from ...core import app, entities as core_entities
-from . import entities
+from . import entities, loader as tools_loader
 from ...plugin import context as plugin_context
+from .loaders import plugin, mcp
 
 
 class ToolManager:
@@ -13,33 +14,26 @@ class ToolManager:
 
     ap: app.Application
 
+    loaders: list[tools_loader.ToolLoader]
+
     def __init__(self, ap: app.Application):
         self.ap = ap
         self.all_functions = []
+        self.loaders = []
 
     async def initialize(self):
-        pass
 
-    async def get_function_and_plugin(
-        self, name: str
-    ) -> typing.Tuple[entities.LLMFunction, plugin_context.BasePlugin]:
-        """获取函数和插件实例"""
-        for plugin in self.ap.plugin_mgr.plugins(
-            enabled=True, status=plugin_context.RuntimeContainerStatus.INITIALIZED
-        ):
-            for function in plugin.content_functions:
-                if function.name == name:
-                    return function, plugin.plugin_inst
-        return None, None
+        for loader_cls in tools_loader.preregistered_loaders:
+            loader_inst = loader_cls(self.ap)
+            await loader_inst.initialize()
+            self.loaders.append(loader_inst)
 
-    async def get_all_functions(self, plugin_enabled: bool=None, plugin_status: plugin_context.RuntimeContainerStatus=None) -> list[entities.LLMFunction]:
+    async def get_all_functions(self, plugin_enabled: bool=None) -> list[entities.LLMFunction]:
         """获取所有函数"""
         all_functions: list[entities.LLMFunction] = []
 
-        for plugin in self.ap.plugin_mgr.plugins(
-            enabled=plugin_enabled, status=plugin_status
-        ):
-            all_functions.extend(plugin.content_functions)
+        for loader in self.loaders:
+            all_functions.extend(await loader.get_tools(plugin_enabled))
 
         return all_functions
 
@@ -102,38 +96,13 @@ class ToolManager:
     ) -> typing.Any:
         """执行函数调用"""
 
-        try:
+        for loader in self.loaders:
+            if await loader.has_tool(name):
+                return await loader.invoke_tool(query, name, parameters)
+        else:
+            raise ValueError(f"未找到工具: {name}")
 
-            function, plugin = await self.get_function_and_plugin(name)
-            if function is None:
-                return None
-
-            parameters = parameters.copy()
-
-            parameters = {"query": query, **parameters}
-
-            return await function.func(plugin, **parameters)
-        except Exception as e:
-            self.ap.logger.error(f"执行函数 {name} 时发生错误: {e}")
-            traceback.print_exc()
-            return f"error occurred when executing function {name}: {e}"
-        finally:
-            plugin = None
-
-            for p in self.ap.plugin_mgr.plugins():
-                if function in p.content_functions:
-                    plugin = p
-                    break
-
-            if plugin is not None:
-
-                await self.ap.ctr_mgr.usage.post_function_record(
-                    plugin={
-                        "name": plugin.plugin_name,
-                        "remote": plugin.plugin_source,
-                        "version": plugin.plugin_version,
-                        "author": plugin.plugin_author,
-                    },
-                    function_name=function.name,
-                    function_description=function.description,
-                )
+    async def shutdown(self):
+        """关闭所有工具"""
+        for loader in self.loaders:
+            await loader.shutdown()

pkg/utils/constants.py

@@ -1,4 +1,4 @@
-semantic_version = "v3.4.10.4"
+semantic_version = "v3.4.11"
 
 debug_mode = False