From 11342e75de10366994b66cad21f31aadde1a53dd Mon Sep 17 00:00:00 2001 From: Junyan Qin Date: Sat, 12 Apr 2025 15:37:15 +0800 Subject: [PATCH] feat: discovering plugins by manifests --- components.yaml | 4 ++ pkg/discover/engine.py | 67 ++++++++++++++++---- pkg/plugin/context.py | 25 +++++--- pkg/plugin/installers/github.py | 4 +- pkg/plugin/loaders/classic.py | 27 ++++---- pkg/plugin/loaders/manifest.py | 95 ++++++++++++++++++++++++++++ pkg/plugin/manager.py | 41 +++++++----- pkg/plugin/setting.py | 2 +- pkg/provider/tools/loaders/plugin.py | 10 +-- 9 files changed, 215 insertions(+), 60 deletions(-) create mode 100644 pkg/plugin/loaders/manifest.py diff --git a/components.yaml b/components.yaml index f4e2b7bc..fc2084c6 100644 --- a/components.yaml +++ b/components.yaml @@ -17,3 +17,7 @@ spec: LLMAPIRequester: fromDirs: - path: pkg/provider/modelmgr/requesters/ + Plugin: + fromDirs: + - path: plugins/ + maxDepth: 2 diff --git a/pkg/discover/engine.py b/pkg/discover/engine.py index 3960661c..51990a3c 100644 --- a/pkg/discover/engine.py +++ b/pkg/discover/engine.py @@ -34,6 +34,7 @@ class I18nString(pydantic.BaseModel): dic['ja_JP'] = self.ja_JP return dic + class Metadata(pydantic.BaseModel): """元数据""" @@ -46,9 +47,18 @@ class Metadata(pydantic.BaseModel): description: typing.Optional[I18nString] = None """描述""" + version: typing.Optional[str] = None + """版本""" + icon: typing.Optional[str] = None """图标""" + author: typing.Optional[str] = None + """作者""" + + repository: typing.Optional[str] = None + """仓库""" + def __init__(self, **kwargs): super().__init__(**kwargs) @@ -96,6 +106,9 @@ class Component(pydantic.BaseModel): rel_path: str """组件清单相对main.py的路径""" + rel_dir: str + """组件清单相对main.py的目录""" + _metadata: Metadata """组件元数据""" @@ -109,12 +122,18 @@ class Component(pydantic.BaseModel): super().__init__( owner=owner, manifest=manifest, - rel_path=rel_path + rel_path=rel_path, + rel_dir=os.path.dirname(rel_path) ) self._metadata = Metadata(**manifest['metadata']) self._spec = manifest['spec'] self._execution = Execution(**manifest['execution']) if 'execution' in manifest else None + @classmethod + def is_component_manifest(cls, manifest: typing.Dict[str, typing.Any]) -> bool: + """判断是否为组件清单""" + return 'apiVersion' in manifest and 'kind' in manifest and 'metadata' in manifest and 'spec' in manifest + @property def kind(self) -> str: """组件类型""" @@ -132,13 +151,12 @@ class Component(pydantic.BaseModel): @property def execution(self) -> Execution: - """组件执行""" + """组件可执行文件信息""" return self._execution def get_python_component_class(self) -> typing.Type[typing.Any]: """获取Python组件类""" - parent_path = os.path.dirname(self.rel_path) - module_path = os.path.join(parent_path, self.execution.python.path) + module_path = os.path.join(self.rel_dir, self.execution.python.path) if module_path.endswith('.py'): module_path = module_path[:-3] module_path = module_path.replace('/', '.').replace('\\', '.') @@ -168,10 +186,12 @@ class ComponentDiscoveryEngine: def __init__(self, ap: app.Application): self.ap = ap - def load_component_manifest(self, path: str, owner: str = 'builtin', no_save: bool = False) -> Component: + def load_component_manifest(self, path: str, owner: str = 'builtin', no_save: bool = False) -> Component | None: """加载组件清单""" with open(path, 'r', encoding='utf-8') as f: manifest = yaml.safe_load(f) + if not Component.is_component_manifest(manifest): + return None comp = Component( owner=owner, manifest=manifest, @@ -183,12 +203,22 @@ class ComponentDiscoveryEngine: self.components[comp.kind].append(comp) return comp - def load_component_manifests_in_dir(self, path: str, owner: str = 'builtin', no_save: bool = False) -> typing.List[Component]: + def load_component_manifests_in_dir(self, path: str, owner: str = 'builtin', no_save: bool = False, max_depth: int = 1) -> typing.List[Component]: """加载目录中的组件清单""" components: typing.List[Component] = [] - for file in os.listdir(path): - if file.endswith('.yaml') or file.endswith('.yml'): - components.append(self.load_component_manifest(os.path.join(path, file), owner, no_save)) + + def recursive_load_component_manifests_in_dir(path: str, depth: int = 1): + if depth > max_depth: + return + for file in os.listdir(path): + if (not os.path.isdir(os.path.join(path, file))) and (file.endswith('.yaml') or file.endswith('.yml')): + comp = self.load_component_manifest(os.path.join(path, file), owner, no_save) + if comp is not None: + components.append(comp) + elif os.path.isdir(os.path.join(path, file)): + recursive_load_component_manifests_in_dir(os.path.join(path, file), depth + 1) + + recursive_load_component_manifests_in_dir(path) return components def load_blueprint_comp_group(self, group: dict, owner: str = 'builtin', no_save: bool = False) -> typing.List[Component]: @@ -196,17 +226,21 @@ class ComponentDiscoveryEngine: components: typing.List[Component] = [] if 'fromFiles' in group: for file in group['fromFiles']: - components.append(self.load_component_manifest(file, owner, no_save)) + comp = self.load_component_manifest(file, owner, no_save) + if comp is not None: + components.append(comp) if 'fromDirs' in group: for dir in group['fromDirs']: path = dir['path'] - # depth = dir['depth'] - components.extend(self.load_component_manifests_in_dir(path, owner, no_save)) + max_depth = dir['maxDepth'] if 'maxDepth' in dir else 1 + components.extend(self.load_component_manifests_in_dir(path, owner, no_save, max_depth)) return components def discover_blueprint(self, blueprint_manifest_path: str, owner: str = 'builtin'): """发现蓝图""" blueprint_manifest = self.load_component_manifest(blueprint_manifest_path, owner, no_save=True) + if blueprint_manifest is None: + raise ValueError(f'Invalid blueprint manifest: {blueprint_manifest_path}') assert blueprint_manifest.kind == 'Blueprint', '`Kind` must be `Blueprint`' components: typing.Dict[str, typing.List[Component]] = {} @@ -223,9 +257,16 @@ class ComponentDiscoveryEngine: return blueprint_manifest, components - def get_components_by_kind(self, kind: str) -> typing.List[Component]: """获取指定类型的组件""" if kind not in self.components: raise ValueError(f'No components found for kind: {kind}') return self.components[kind] + + def find_components(self, kind: str, component_list: typing.List[Component]) -> typing.List[Component]: + """查找组件""" + result: typing.List[Component] = [] + for component in component_list: + if component.kind == kind: + result.append(component) + return result diff --git a/pkg/plugin/context.py b/pkg/plugin/context.py index 7a9be2a1..16fe414d 100644 --- a/pkg/plugin/context.py +++ b/pkg/plugin/context.py @@ -8,6 +8,7 @@ import enum from . import events from ..provider.tools import entities as tools_entities from ..core import app +from ..discover import engine as discover_engine from ..platform.types import message as platform_message from ..platform import adapter as platform_adapter @@ -308,7 +309,10 @@ class RuntimeContainer(pydantic.BaseModel): plugin_name: str """插件名称""" - plugin_description: str + plugin_label: discover_engine.I18nString + """插件标签""" + + plugin_description: discover_engine.I18nString """插件描述""" plugin_version: str @@ -317,7 +321,7 @@ class RuntimeContainer(pydantic.BaseModel): plugin_author: str """插件作者""" - plugin_source: str + plugin_repository: str """插件源码地址""" main_file: str @@ -343,7 +347,7 @@ class RuntimeContainer(pydantic.BaseModel): ]] = {} """事件处理器""" - content_functions: list[tools_entities.LLMFunction] = [] + tools: list[tools_entities.LLMFunction] = [] """内容函数""" status: RuntimeContainerStatus = RuntimeContainerStatus.MOUNTED @@ -355,10 +359,10 @@ class RuntimeContainer(pydantic.BaseModel): def to_setting_dict(self): return { 'name': self.plugin_name, - 'description': self.plugin_description, + 'description': self.plugin_description.to_dict(), 'version': self.plugin_version, 'author': self.plugin_author, - 'source': self.plugin_source, + 'source': self.plugin_repository, 'main_file': self.main_file, 'pkg_path': self.pkg_path, 'priority': self.priority, @@ -369,17 +373,18 @@ class RuntimeContainer(pydantic.BaseModel): self, setting: dict ): - self.plugin_source = setting['source'] + self.plugin_repository = setting['source'] self.priority = setting['priority'] self.enabled = setting['enabled'] def model_dump(self, *args, **kwargs): return { 'name': self.plugin_name, - 'description': self.plugin_description, + 'label': self.plugin_label.to_dict(), + 'description': self.plugin_description.to_dict(), 'version': self.plugin_version, 'author': self.plugin_author, - 'source': self.plugin_source, + 'repository': self.plugin_repository, 'main_file': self.main_file, 'pkg_path': self.pkg_path, 'enabled': self.enabled, @@ -388,7 +393,7 @@ class RuntimeContainer(pydantic.BaseModel): event_name.__name__: handler.__name__ for event_name, handler in self.event_handlers.items() }, - 'content_functions': [ + 'tools': [ { 'name': function.name, 'human_desc': function.human_desc, @@ -396,7 +401,7 @@ class RuntimeContainer(pydantic.BaseModel): 'parameters': function.parameters, 'func': function.func.__name__, } - for function in self.content_functions + for function in self.tools ], 'status': self.status.value, } diff --git a/pkg/plugin/installers/github.py b/pkg/plugin/installers/github.py index 039ff196..77ee7490 100644 --- a/pkg/plugin/installers/github.py +++ b/pkg/plugin/installers/github.py @@ -129,8 +129,8 @@ class GitHubRepoInstaller(installer.PluginInstaller): if plugin_container is None: raise errors.PluginInstallerError('插件不存在或未成功加载') else: - if plugin_container.plugin_source: - plugin_source = plugin_container.plugin_source + if plugin_container.plugin_repository: + plugin_source = plugin_container.plugin_repository task_context.trace("转交安装任务.", "update-plugin") await self.install_plugin(plugin_source, task_context) else: diff --git a/pkg/plugin/loaders/classic.py b/pkg/plugin/loaders/classic.py index b3710c9e..23a107d4 100644 --- a/pkg/plugin/loaders/classic.py +++ b/pkg/plugin/loaders/classic.py @@ -9,7 +9,7 @@ from .. import loader, events, context, models from ...core import entities as core_entities from ...provider.tools import entities as tools_entities from ...utils import funcschema - +from ...discover import engine as discover_engine class PluginLoader(loader.PluginLoader): """加载 plugins/ 目录下的插件""" @@ -31,13 +31,6 @@ class PluginLoader(loader.PluginLoader): async def initialize(self): """初始化""" - setattr(models, 'register', self.register) - setattr(models, 'on', self.on) - setattr(models, 'func', self.func) - - setattr(context, 'register', self.register) - setattr(context, 'handler', self.handler) - setattr(context, 'llm_func', self.llm_func) def register( self, @@ -49,14 +42,15 @@ class PluginLoader(loader.PluginLoader): self.ap.logger.debug(f'注册插件 {name} {version} by {author}') container = context.RuntimeContainer( plugin_name=name, - plugin_description=description, + plugin_label=discover_engine.I18nString(en_US=name, zh_CN=name), + plugin_description=discover_engine.I18nString(en_US=description, zh_CN=description), plugin_version=version, plugin_author=author, - plugin_source='', + plugin_repository='', pkg_path=self._current_pkg_path, main_file=self._current_module_path, event_handlers={}, - content_functions=[], + tools=[], ) self._current_container = container @@ -126,7 +120,7 @@ class PluginLoader(loader.PluginLoader): func=handler, ) - self._current_container.content_functions.append(llm_function) + self._current_container.tools.append(llm_function) return func @@ -165,7 +159,7 @@ class PluginLoader(loader.PluginLoader): func=func, ) - self._current_container.content_functions.append(llm_function) + self._current_container.tools.append(llm_function) return func @@ -205,4 +199,11 @@ class PluginLoader(loader.PluginLoader): async def load_plugins(self): """加载插件 """ + setattr(models, 'register', self.register) + setattr(models, 'on', self.on) + setattr(models, 'func', self.func) + + setattr(context, 'register', self.register) + setattr(context, 'handler', self.handler) + setattr(context, 'llm_func', self.llm_func) await self._walk_plugin_path(__import__("plugins", fromlist=[""])) diff --git a/pkg/plugin/loaders/manifest.py b/pkg/plugin/loaders/manifest.py new file mode 100644 index 00000000..2f7b83f2 --- /dev/null +++ b/pkg/plugin/loaders/manifest.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import typing +import abc +import os +import traceback + +from ...core import app +from .. import context, events, models +from .. import loader +from ...utils import funcschema +from ...provider.tools import entities as tools_entities + + +class PluginManifestLoader(loader.PluginLoader): + """通过插件清单发现插件""" + + _current_container: context.RuntimeContainer = None + + def __init__(self, ap: app.Application): + super().__init__(ap) + + def handler( + self, + event: typing.Type[events.BaseEventModel] + ) -> typing.Callable[[typing.Callable], typing.Callable]: + """注册事件处理器""" + self.ap.logger.debug(f'注册事件处理器 {event.__name__}') + def wrapper(func: typing.Callable) -> typing.Callable: + + self._current_container.event_handlers[event] = func + + return func + + return wrapper + + def llm_func( + self, + name: str=None, + ) -> typing.Callable: + """注册内容函数""" + self.ap.logger.debug(f'注册内容函数 {name}') + def wrapper(func: typing.Callable) -> typing.Callable: + + function_schema = funcschema.get_func_schema(func) + function_name = self._current_container.plugin_name + '-' + (func.__name__ if name is None else name) + + llm_function = tools_entities.LLMFunction( + name=function_name, + human_desc='', + description=function_schema['description'], + parameters=function_schema['parameters'], + func=func, + ) + + self._current_container.tools.append(llm_function) + + return func + + return wrapper + + async def load_plugins(self): + """加载插件""" + setattr(context, 'handler', self.handler) + setattr(context, 'llm_func', self.llm_func) + + plugin_manifests = self.ap.discover.get_components_by_kind('Plugin') + + for plugin_manifest in plugin_manifests: + try: + current_plugin_container = context.RuntimeContainer( + plugin_name=plugin_manifest.metadata.name, + plugin_label=plugin_manifest.metadata.label, + plugin_description=plugin_manifest.metadata.description, + plugin_version=plugin_manifest.metadata.version, + plugin_author=plugin_manifest.metadata.author, + plugin_repository=plugin_manifest.metadata.repository, + main_file=os.path.join(plugin_manifest.rel_dir, plugin_manifest.execution.python.path), + pkg_path=plugin_manifest.rel_dir, + event_handlers={}, + tools=[], + ) + + self._current_container = current_plugin_container + + # extract the plugin class + # this step will load the plugin module, + # so the event handlers and tools will be registered + plugin_class = plugin_manifest.get_python_component_class() + current_plugin_container.plugin_class = plugin_class + + self.plugins.append(current_plugin_container) + except Exception as e: + self.ap.logger.error(f'加载插件 {plugin_manifest.metadata.name} 时发生错误') + traceback.print_exc() diff --git a/pkg/plugin/manager.py b/pkg/plugin/manager.py index 2b8e887d..cdae7b19 100644 --- a/pkg/plugin/manager.py +++ b/pkg/plugin/manager.py @@ -5,7 +5,7 @@ import traceback from ..core import app, taskmgr from . import context, loader, events, installer, setting, models -from .loaders import classic +from .loaders import classic, manifest from .installers import github @@ -14,7 +14,7 @@ class PluginManager: ap: app.Application - loader: loader.PluginLoader + loaders: list[loader.PluginLoader] installer: installer.PluginInstaller @@ -22,6 +22,8 @@ class PluginManager: api_host: context.APIHost + plugin_containers: list[context.RuntimeContainer] + def plugins( self, enabled: bool=None, @@ -29,7 +31,7 @@ class PluginManager: ) -> list[context.RuntimeContainer]: """获取插件列表 """ - plugins = self.loader.plugins + plugins = self.plugin_containers if enabled is not None: plugins = [plugin for plugin in plugins if plugin.enabled == enabled] @@ -41,13 +43,18 @@ class PluginManager: def __init__(self, ap: app.Application): self.ap = ap - self.loader = classic.PluginLoader(ap) + self.loaders = [ + classic.PluginLoader(ap), + manifest.PluginManifestLoader(ap), + ] self.installer = github.GitHubRepoInstaller(ap) self.setting = setting.SettingManager(ap) self.api_host = context.APIHost(ap) + self.plugin_containers = [] async def initialize(self): - await self.loader.initialize() + for loader in self.loaders: + await loader.initialize() await self.installer.initialize() await self.setting.initialize() await self.api_host.initialize() @@ -55,14 +62,16 @@ class PluginManager: setattr(models, 'require_ver', self.api_host.require_ver) async def load_plugins(self): - await self.loader.load_plugins() + for loader in self.loaders: + await loader.load_plugins() + self.plugin_containers.extend(loader.plugins) - await self.setting.sync_setting(self.loader.plugins) + await self.setting.sync_setting(self.plugin_containers) # 按优先级倒序 - self.loader.plugins.sort(key=lambda x: x.priority, reverse=True) + self.plugin_containers.sort(key=lambda x: x.priority, reverse=True) - self.ap.logger.debug(f'优先级排序后的插件列表 {self.loader.plugins}') + self.ap.logger.debug(f'优先级排序后的插件列表 {self.plugin_containers}') async def initialize_plugin(self, plugin: context.RuntimeContainer): self.ap.logger.debug(f'初始化插件 {plugin.plugin_name}') @@ -147,7 +156,7 @@ class PluginManager: await self.ap.ctr_mgr.plugin.post_remove_record( { "name": plugin_name, - "remote": plugin_container.plugin_source, + "remote": plugin_container.plugin_repository, "author": plugin_container.plugin_author, "version": plugin_container.plugin_version } @@ -171,7 +180,7 @@ class PluginManager: await self.ap.ctr_mgr.plugin.post_update_record( plugin={ "name": plugin_name, - "remote": plugin_container.plugin_source, + "remote": plugin_container.plugin_repository, "author": plugin_container.plugin_author, "version": plugin_container.plugin_version }, @@ -238,7 +247,7 @@ class PluginManager: plugins_info: list[dict] = [ { 'name': plugin.plugin_name, - 'remote': plugin.plugin_source, + 'remote': plugin.plugin_repository, 'version': plugin.plugin_version, 'author': plugin.plugin_author } for plugin in emitted_plugins @@ -266,7 +275,7 @@ class PluginManager: plugin.enabled = new_status - await self.setting.dump_container_setting(self.loader.plugins) + await self.setting.dump_container_setting(self.plugin_containers) break @@ -280,11 +289,11 @@ class PluginManager: plugin_name = plugin.get('name') plugin_priority = plugin.get('priority') - for plugin in self.loader.plugins: + for plugin in self.plugin_containers: if plugin.plugin_name == plugin_name: plugin.priority = plugin_priority break - self.loader.plugins.sort(key=lambda x: x.priority, reverse=True) + self.plugin_containers.sort(key=lambda x: x.priority, reverse=True) - await self.setting.dump_container_setting(self.loader.plugins) + await self.setting.dump_container_setting(self.plugin_containers) diff --git a/pkg/plugin/setting.py b/pkg/plugin/setting.py index bd50603f..b5c5c06f 100644 --- a/pkg/plugin/setting.py +++ b/pkg/plugin/setting.py @@ -36,7 +36,7 @@ class SettingManager: if plugin_container.pkg_path == value['pkg_path']: matched = True - plugin_container.plugin_source = value['source'] + plugin_container.plugin_repository = value['source'] break if not matched: diff --git a/pkg/provider/tools/loaders/plugin.py b/pkg/provider/tools/loaders/plugin.py index 08211334..5b964556 100644 --- a/pkg/provider/tools/loaders/plugin.py +++ b/pkg/provider/tools/loaders/plugin.py @@ -23,7 +23,7 @@ class PluginToolLoader(loader.ToolLoader): for plugin in self.ap.plugin_mgr.plugins( enabled=enabled, status=plugin_context.RuntimeContainerStatus.INITIALIZED ): - all_functions.extend(plugin.content_functions) + all_functions.extend(plugin.tools) return all_functions @@ -32,7 +32,7 @@ class PluginToolLoader(loader.ToolLoader): for plugin in self.ap.plugin_mgr.plugins( enabled=True, status=plugin_context.RuntimeContainerStatus.INITIALIZED ): - for function in plugin.content_functions: + for function in plugin.tools: if function.name == name: return True return False @@ -44,7 +44,7 @@ class PluginToolLoader(loader.ToolLoader): for plugin in self.ap.plugin_mgr.plugins( enabled=True, status=plugin_context.RuntimeContainerStatus.INITIALIZED ): - for function in plugin.content_functions: + for function in plugin.tools: if function.name == name: return function, plugin.plugin_inst return None, None @@ -70,7 +70,7 @@ class PluginToolLoader(loader.ToolLoader): plugin = None for p in self.ap.plugin_mgr.plugins(): - if function in p.content_functions: + if function in p.tools: plugin = p break @@ -79,7 +79,7 @@ class PluginToolLoader(loader.ToolLoader): await self.ap.ctr_mgr.usage.post_function_record( plugin={ "name": plugin.plugin_name, - "remote": plugin.plugin_source, + "remote": plugin.plugin_repository, "version": plugin.plugin_version, "author": plugin.plugin_author, },