feat: discovering plugins by manifests

components.yaml

@@ -17,3 +17,7 @@ spec:
     LLMAPIRequester:
       fromDirs:
         - path: pkg/provider/modelmgr/requesters/
+    Plugin:
+      fromDirs:
+        - path: plugins/
+          maxDepth: 2

pkg/discover/engine.py

@@ -34,6 +34,7 @@ class I18nString(pydantic.BaseModel):
             dic['ja_JP'] = self.ja_JP
         return dic
 
+
 class Metadata(pydantic.BaseModel):
     """元数据"""
 
@@ -46,9 +47,18 @@ class Metadata(pydantic.BaseModel):
     description: typing.Optional[I18nString] = None
     """描述"""
 
+    version: typing.Optional[str] = None
+    """版本"""
+
     icon: typing.Optional[str] = None
     """图标"""
 
+    author: typing.Optional[str] = None
+    """作者"""
+
+    repository: typing.Optional[str] = None
+    """仓库"""
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         
@@ -96,6 +106,9 @@ class Component(pydantic.BaseModel):
     rel_path: str
     """组件清单相对main.py的路径"""
 
+    rel_dir: str
+    """组件清单相对main.py的目录"""
+
     _metadata: Metadata
     """组件元数据"""
 
@@ -109,12 +122,18 @@ class Component(pydantic.BaseModel):
         super().__init__(
             owner=owner,
             manifest=manifest,
-            rel_path=rel_path
+            rel_path=rel_path,
+            rel_dir=os.path.dirname(rel_path)
         )
         self._metadata = Metadata(**manifest['metadata'])
         self._spec = manifest['spec']
         self._execution = Execution(**manifest['execution']) if 'execution' in manifest else None
 
+    @classmethod
+    def is_component_manifest(cls, manifest: typing.Dict[str, typing.Any]) -> bool:
+        """判断是否为组件清单"""
+        return 'apiVersion' in manifest and 'kind' in manifest and 'metadata' in manifest and 'spec' in manifest
+
     @property
     def kind(self) -> str:
         """组件类型"""
@@ -132,13 +151,12 @@ class Component(pydantic.BaseModel):
     
     @property
     def execution(self) -> Execution:
-        """组件执行"""
+        """组件可执行文件信息"""
         return self._execution
     
     def get_python_component_class(self) -> typing.Type[typing.Any]:
         """获取Python组件类"""
-        parent_path = os.path.dirname(self.rel_path)
+        module_path = os.path.join(self.rel_dir, self.execution.python.path)
-        module_path = os.path.join(parent_path, self.execution.python.path)
         if module_path.endswith('.py'):
             module_path = module_path[:-3]
         module_path = module_path.replace('/', '.').replace('\\', '.')
@@ -168,10 +186,12 @@ class ComponentDiscoveryEngine:
     def __init__(self, ap: app.Application):
         self.ap = ap
 
-    def load_component_manifest(self, path: str, owner: str = 'builtin', no_save: bool = False) -> Component:
+    def load_component_manifest(self, path: str, owner: str = 'builtin', no_save: bool = False) -> Component | None:
         """加载组件清单"""
         with open(path, 'r', encoding='utf-8') as f:
             manifest = yaml.safe_load(f)
+            if not Component.is_component_manifest(manifest):
+                return None
             comp = Component(
                 owner=owner,
                 manifest=manifest,
@@ -183,12 +203,22 @@ class ComponentDiscoveryEngine:
                 self.components[comp.kind].append(comp)
             return comp
         
-    def load_component_manifests_in_dir(self, path: str, owner: str = 'builtin', no_save: bool = False) -> typing.List[Component]:
+    def load_component_manifests_in_dir(self, path: str, owner: str = 'builtin', no_save: bool = False, max_depth: int = 1) -> typing.List[Component]:
         """加载目录中的组件清单"""
         components: typing.List[Component] = []
-        for file in os.listdir(path):
+
-            if file.endswith('.yaml') or file.endswith('.yml'):
+        def recursive_load_component_manifests_in_dir(path: str, depth: int = 1):
-                components.append(self.load_component_manifest(os.path.join(path, file), owner, no_save))
+            if depth > max_depth:
+                return
+            for file in os.listdir(path):
+                if (not os.path.isdir(os.path.join(path, file))) and (file.endswith('.yaml') or file.endswith('.yml')):
+                    comp = self.load_component_manifest(os.path.join(path, file), owner, no_save)
+                    if comp is not None:
+                        components.append(comp)
+                elif os.path.isdir(os.path.join(path, file)):
+                    recursive_load_component_manifests_in_dir(os.path.join(path, file), depth + 1)
+
+        recursive_load_component_manifests_in_dir(path)
         return components
     
     def load_blueprint_comp_group(self, group: dict, owner: str = 'builtin', no_save: bool = False) -> typing.List[Component]:
@@ -196,17 +226,21 @@ class ComponentDiscoveryEngine:
         components: typing.List[Component] = []
         if 'fromFiles' in group:
             for file in group['fromFiles']:
-                components.append(self.load_component_manifest(file, owner, no_save))
+                comp = self.load_component_manifest(file, owner, no_save)
+                if comp is not None:
+                    components.append(comp)
         if 'fromDirs' in group:
             for dir in group['fromDirs']:
                 path = dir['path']
-                # depth = dir['depth']
+                max_depth = dir['maxDepth'] if 'maxDepth' in dir else 1
-                components.extend(self.load_component_manifests_in_dir(path, owner, no_save))
+                components.extend(self.load_component_manifests_in_dir(path, owner, no_save, max_depth))
         return components
 
     def discover_blueprint(self, blueprint_manifest_path: str, owner: str = 'builtin'):
         """发现蓝图"""
         blueprint_manifest = self.load_component_manifest(blueprint_manifest_path, owner, no_save=True)
+        if blueprint_manifest is None:
+            raise ValueError(f'Invalid blueprint manifest: {blueprint_manifest_path}')
         assert blueprint_manifest.kind == 'Blueprint', '`Kind` must be `Blueprint`'
         components: typing.Dict[str, typing.List[Component]] = {}
 
@@ -223,9 +257,16 @@ class ComponentDiscoveryEngine:
 
         return blueprint_manifest, components
 
-
     def get_components_by_kind(self, kind: str) -> typing.List[Component]:
         """获取指定类型的组件"""
         if kind not in self.components:
             raise ValueError(f'No components found for kind: {kind}')
         return self.components[kind]
+
+    def find_components(self, kind: str, component_list: typing.List[Component]) -> typing.List[Component]:
+        """查找组件"""
+        result: typing.List[Component] = []
+        for component in component_list:
+            if component.kind == kind:
+                result.append(component)
+        return result

pkg/plugin/context.py

@@ -8,6 +8,7 @@ import enum
 from . import events
 from ..provider.tools import entities as tools_entities
 from ..core import app
+from ..discover import engine as discover_engine
 from ..platform.types import message as platform_message
 from ..platform import adapter as platform_adapter
 
@@ -308,7 +309,10 @@ class RuntimeContainer(pydantic.BaseModel):
     plugin_name: str
     """插件名称"""
 
-    plugin_description: str
+    plugin_label: discover_engine.I18nString
+    """插件标签"""
+
+    plugin_description: discover_engine.I18nString
     """插件描述"""
 
     plugin_version: str
@@ -317,7 +321,7 @@ class RuntimeContainer(pydantic.BaseModel):
     plugin_author: str
     """插件作者"""
 
-    plugin_source: str
+    plugin_repository: str
     """插件源码地址"""
 
     main_file: str
@@ -343,7 +347,7 @@ class RuntimeContainer(pydantic.BaseModel):
     ]] = {}
     """事件处理器"""
 
-    content_functions: list[tools_entities.LLMFunction] = []
+    tools: list[tools_entities.LLMFunction] = []
     """内容函数"""
 
     status: RuntimeContainerStatus = RuntimeContainerStatus.MOUNTED
@@ -355,10 +359,10 @@ class RuntimeContainer(pydantic.BaseModel):
     def to_setting_dict(self):
         return {
             'name': self.plugin_name,
-            'description': self.plugin_description,
+            'description': self.plugin_description.to_dict(),
             'version': self.plugin_version,
             'author': self.plugin_author,
-            'source': self.plugin_source,
+            'source': self.plugin_repository,
             'main_file': self.main_file,
             'pkg_path': self.pkg_path,
             'priority': self.priority,
@@ -369,17 +373,18 @@ class RuntimeContainer(pydantic.BaseModel):
         self,
         setting: dict
     ):
-        self.plugin_source = setting['source']
+        self.plugin_repository = setting['source']
         self.priority = setting['priority']
         self.enabled = setting['enabled']
 
     def model_dump(self, *args, **kwargs):
         return {
             'name': self.plugin_name,
-            'description': self.plugin_description,
+            'label': self.plugin_label.to_dict(),
+            'description': self.plugin_description.to_dict(),
             'version': self.plugin_version,
             'author': self.plugin_author,
-            'source': self.plugin_source,
+            'repository': self.plugin_repository,
             'main_file': self.main_file,
             'pkg_path': self.pkg_path,
             'enabled': self.enabled,
@@ -388,7 +393,7 @@ class RuntimeContainer(pydantic.BaseModel):
                 event_name.__name__: handler.__name__
                 for event_name, handler in self.event_handlers.items()
             },
-            'content_functions': [
+            'tools': [
                 {
                     'name': function.name,
                     'human_desc': function.human_desc,
@@ -396,7 +401,7 @@ class RuntimeContainer(pydantic.BaseModel):
                     'parameters': function.parameters,
                     'func': function.func.__name__,
                 }
-                for function in self.content_functions
+                for function in self.tools
             ],
             'status': self.status.value,
         }

pkg/plugin/installers/github.py

@@ -129,8 +129,8 @@ class GitHubRepoInstaller(installer.PluginInstaller):
         if plugin_container is None:
             raise errors.PluginInstallerError('插件不存在或未成功加载')
         else:
-            if plugin_container.plugin_source:
+            if plugin_container.plugin_repository:
-                plugin_source = plugin_container.plugin_source
+                plugin_source = plugin_container.plugin_repository
                 task_context.trace("转交安装任务.", "update-plugin")
                 await self.install_plugin(plugin_source, task_context)
             else:

pkg/plugin/loaders/classic.py

@@ -9,7 +9,7 @@ from .. import loader, events, context, models
 from ...core import entities as core_entities
 from ...provider.tools import entities as tools_entities
 from ...utils import funcschema
-
+from ...discover import engine as discover_engine
 
 class PluginLoader(loader.PluginLoader):
     """加载 plugins/ 目录下的插件"""
@@ -31,13 +31,6 @@ class PluginLoader(loader.PluginLoader):
 
     async def initialize(self):
         """初始化"""
-        setattr(models, 'register', self.register)
-        setattr(models, 'on', self.on)
-        setattr(models, 'func', self.func)
-
-        setattr(context, 'register', self.register)
-        setattr(context, 'handler', self.handler)
-        setattr(context, 'llm_func', self.llm_func)
 
     def register(
         self,
@@ -49,14 +42,15 @@ class PluginLoader(loader.PluginLoader):
         self.ap.logger.debug(f'注册插件 {name} {version} by {author}')
         container = context.RuntimeContainer(
             plugin_name=name,
-            plugin_description=description,
+            plugin_label=discover_engine.I18nString(en_US=name, zh_CN=name),
+            plugin_description=discover_engine.I18nString(en_US=description, zh_CN=description),
             plugin_version=version,
             plugin_author=author,
-            plugin_source='',
+            plugin_repository='',
             pkg_path=self._current_pkg_path,
             main_file=self._current_module_path,
             event_handlers={},
-            content_functions=[],
+            tools=[],
         )
 
         self._current_container = container
@@ -126,7 +120,7 @@ class PluginLoader(loader.PluginLoader):
                 func=handler,
             )
 
-            self._current_container.content_functions.append(llm_function)
+            self._current_container.tools.append(llm_function)
 
             return func
         
@@ -165,7 +159,7 @@ class PluginLoader(loader.PluginLoader):
                 func=func,
             )
 
-            self._current_container.content_functions.append(llm_function)
+            self._current_container.tools.append(llm_function)
 
             return func
         
@@ -205,4 +199,11 @@ class PluginLoader(loader.PluginLoader):
     async def load_plugins(self):
         """加载插件
         """
+        setattr(models, 'register', self.register)
+        setattr(models, 'on', self.on)
+        setattr(models, 'func', self.func)
+
+        setattr(context, 'register', self.register)
+        setattr(context, 'handler', self.handler)
+        setattr(context, 'llm_func', self.llm_func)
         await self._walk_plugin_path(__import__("plugins", fromlist=[""]))

pkg/plugin/loaders/manifest.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import typing
+import abc
+import os
+import traceback
+
+from ...core import app
+from .. import context, events, models
+from .. import loader
+from ...utils import funcschema
+from ...provider.tools import entities as tools_entities
+
+
+class PluginManifestLoader(loader.PluginLoader):
+    """通过插件清单发现插件"""
+
+    _current_container: context.RuntimeContainer = None
+
+    def __init__(self, ap: app.Application):
+        super().__init__(ap)
+
+    def handler(
+        self,
+        event: typing.Type[events.BaseEventModel]
+    ) -> typing.Callable[[typing.Callable], typing.Callable]:
+        """注册事件处理器"""
+        self.ap.logger.debug(f'注册事件处理器 {event.__name__}')
+        def wrapper(func: typing.Callable) -> typing.Callable:
+            
+            self._current_container.event_handlers[event] = func
+
+            return func
+
+        return wrapper
+
+    def llm_func(
+        self,
+        name: str=None,
+    ) -> typing.Callable:
+        """注册内容函数"""
+        self.ap.logger.debug(f'注册内容函数 {name}')
+        def wrapper(func: typing.Callable) -> typing.Callable:
+            
+            function_schema = funcschema.get_func_schema(func)
+            function_name = self._current_container.plugin_name + '-' + (func.__name__ if name is None else name)
+
+            llm_function = tools_entities.LLMFunction(
+                name=function_name,
+                human_desc='',
+                description=function_schema['description'],
+                parameters=function_schema['parameters'],
+                func=func,
+            )
+
+            self._current_container.tools.append(llm_function)
+
+            return func
+        
+        return wrapper
+
+    async def load_plugins(self):
+        """加载插件"""
+        setattr(context, 'handler', self.handler)
+        setattr(context, 'llm_func', self.llm_func)
+
+        plugin_manifests = self.ap.discover.get_components_by_kind('Plugin')
+
+        for plugin_manifest in plugin_manifests:
+            try:
+                current_plugin_container = context.RuntimeContainer(
+                    plugin_name=plugin_manifest.metadata.name,
+                    plugin_label=plugin_manifest.metadata.label,
+                    plugin_description=plugin_manifest.metadata.description,
+                    plugin_version=plugin_manifest.metadata.version,
+                    plugin_author=plugin_manifest.metadata.author,
+                    plugin_repository=plugin_manifest.metadata.repository,
+                    main_file=os.path.join(plugin_manifest.rel_dir, plugin_manifest.execution.python.path),
+                    pkg_path=plugin_manifest.rel_dir,
+                    event_handlers={},
+                    tools=[],
+                )
+
+                self._current_container = current_plugin_container
+
+                # extract the plugin class
+                # this step will load the plugin module,
+                # so the event handlers and tools will be registered
+                plugin_class = plugin_manifest.get_python_component_class()
+                current_plugin_container.plugin_class = plugin_class
+
+                self.plugins.append(current_plugin_container)
+            except Exception as e:
+                self.ap.logger.error(f'加载插件 {plugin_manifest.metadata.name} 时发生错误')
+                traceback.print_exc()

pkg/plugin/manager.py

@@ -5,7 +5,7 @@ import traceback
 
 from ..core import app, taskmgr
 from . import context, loader, events, installer, setting, models
-from .loaders import classic
+from .loaders import classic, manifest
 from .installers import github
 
 
@@ -14,7 +14,7 @@ class PluginManager:
 
     ap: app.Application
 
-    loader: loader.PluginLoader
+    loaders: list[loader.PluginLoader]
 
     installer: installer.PluginInstaller
 
@@ -22,6 +22,8 @@ class PluginManager:
 
     api_host: context.APIHost
 
+    plugin_containers: list[context.RuntimeContainer]
+
     def plugins(
         self,
         enabled: bool=None,
@@ -29,7 +31,7 @@ class PluginManager:
     ) -> list[context.RuntimeContainer]:
         """获取插件列表
         """
-        plugins = self.loader.plugins
+        plugins = self.plugin_containers
 
         if enabled is not None:
             plugins = [plugin for plugin in plugins if plugin.enabled == enabled]
@@ -41,13 +43,18 @@ class PluginManager:
 
     def __init__(self, ap: app.Application):
         self.ap = ap
-        self.loader = classic.PluginLoader(ap)
+        self.loaders = [
+            classic.PluginLoader(ap),
+            manifest.PluginManifestLoader(ap),
+        ]
         self.installer = github.GitHubRepoInstaller(ap)
         self.setting = setting.SettingManager(ap)
         self.api_host = context.APIHost(ap)
+        self.plugin_containers = []
 
     async def initialize(self):
-        await self.loader.initialize()
+        for loader in self.loaders:
+            await loader.initialize()
         await self.installer.initialize()
         await self.setting.initialize()
         await self.api_host.initialize()
@@ -55,14 +62,16 @@ class PluginManager:
         setattr(models, 'require_ver', self.api_host.require_ver)
 
     async def load_plugins(self):
-        await self.loader.load_plugins()
+        for loader in self.loaders:
+            await loader.load_plugins()
+            self.plugin_containers.extend(loader.plugins)
 
-        await self.setting.sync_setting(self.loader.plugins)
+        await self.setting.sync_setting(self.plugin_containers)
 
         # 按优先级倒序
-        self.loader.plugins.sort(key=lambda x: x.priority, reverse=True)
+        self.plugin_containers.sort(key=lambda x: x.priority, reverse=True)
 
-        self.ap.logger.debug(f'优先级排序后的插件列表 {self.loader.plugins}')
+        self.ap.logger.debug(f'优先级排序后的插件列表 {self.plugin_containers}')
 
     async def initialize_plugin(self, plugin: context.RuntimeContainer):
         self.ap.logger.debug(f'初始化插件 {plugin.plugin_name}')
@@ -147,7 +156,7 @@ class PluginManager:
         await self.ap.ctr_mgr.plugin.post_remove_record(
             {
                 "name": plugin_name,
-                "remote": plugin_container.plugin_source,
+                "remote": plugin_container.plugin_repository,
                 "author": plugin_container.plugin_author,
                 "version": plugin_container.plugin_version
             }
@@ -171,7 +180,7 @@ class PluginManager:
         await self.ap.ctr_mgr.plugin.post_update_record(
             plugin={
                 "name": plugin_name,
-                "remote": plugin_container.plugin_source,
+                "remote": plugin_container.plugin_repository,
                 "author": plugin_container.plugin_author,
                 "version": plugin_container.plugin_version
             },
@@ -238,7 +247,7 @@ class PluginManager:
             plugins_info: list[dict] = [
                 {
                     'name': plugin.plugin_name,
-                    'remote': plugin.plugin_source,
+                    'remote': plugin.plugin_repository,
                     'version': plugin.plugin_version,
                     'author': plugin.plugin_author
                 } for plugin in emitted_plugins
@@ -266,7 +275,7 @@ class PluginManager:
 
                     plugin.enabled = new_status
                     
-                    await self.setting.dump_container_setting(self.loader.plugins)
+                    await self.setting.dump_container_setting(self.plugin_containers)
 
                     break
 
@@ -280,11 +289,11 @@ class PluginManager:
             plugin_name = plugin.get('name')
             plugin_priority = plugin.get('priority')
 
-            for plugin in self.loader.plugins:
+            for plugin in self.plugin_containers:
                 if plugin.plugin_name == plugin_name:
                     plugin.priority = plugin_priority
                     break
 
-        self.loader.plugins.sort(key=lambda x: x.priority, reverse=True)
+        self.plugin_containers.sort(key=lambda x: x.priority, reverse=True)
 
-        await self.setting.dump_container_setting(self.loader.plugins)
+        await self.setting.dump_container_setting(self.plugin_containers)

pkg/plugin/setting.py

@@ -36,7 +36,7 @@ class SettingManager:
                     if plugin_container.pkg_path == value['pkg_path']:
                         matched = True
 
-                        plugin_container.plugin_source = value['source']
+                        plugin_container.plugin_repository = value['source']
                         break
 
                 if not matched:

pkg/provider/tools/loaders/plugin.py

@@ -23,7 +23,7 @@ class PluginToolLoader(loader.ToolLoader):
         for plugin in self.ap.plugin_mgr.plugins(
             enabled=enabled, status=plugin_context.RuntimeContainerStatus.INITIALIZED
         ):
-            all_functions.extend(plugin.content_functions)
+            all_functions.extend(plugin.tools)
 
         return all_functions
 
@@ -32,7 +32,7 @@ class PluginToolLoader(loader.ToolLoader):
         for plugin in self.ap.plugin_mgr.plugins(
             enabled=True, status=plugin_context.RuntimeContainerStatus.INITIALIZED
         ):
-            for function in plugin.content_functions:
+            for function in plugin.tools:
                 if function.name == name:
                     return True
         return False
@@ -44,7 +44,7 @@ class PluginToolLoader(loader.ToolLoader):
         for plugin in self.ap.plugin_mgr.plugins(
             enabled=True, status=plugin_context.RuntimeContainerStatus.INITIALIZED
         ):
-            for function in plugin.content_functions:
+            for function in plugin.tools:
                 if function.name == name:
                     return function, plugin.plugin_inst
         return None, None
@@ -70,7 +70,7 @@ class PluginToolLoader(loader.ToolLoader):
             plugin = None
 
             for p in self.ap.plugin_mgr.plugins():
-                if function in p.content_functions:
+                if function in p.tools:
                     plugin = p
                     break
 
@@ -79,7 +79,7 @@ class PluginToolLoader(loader.ToolLoader):
                 await self.ap.ctr_mgr.usage.post_function_record(
                     plugin={
                         "name": plugin.plugin_name,
-                        "remote": plugin.plugin_source,
+                        "remote": plugin.plugin_repository,
                         "version": plugin.plugin_version,
                         "author": plugin.plugin_author,
                     },