feat: claude 支持视觉

pkg/provider/modelmgr/apis/anthropicmsgs.py

@@ -11,6 +11,7 @@ from .. import api, entities, errors
 from ....core import entities as core_entities
 from ... import entities as llm_entities
 from ...tools import entities as tools_entities
+from ....utils import image
 
 
 @api.requester_class("anthropic-messages")
@@ -54,29 +55,45 @@ class AnthropicMessages(api.LLMAPIRequester):
             and isinstance(system_role_message.content, str):
             args['system'] = system_role_message.content
 
-        # 其他消息
-        # req_messages = [
-        #     m.dict(exclude_none=True) for m in messages \
-        #         if (isinstance(m.content, str) and m.content.strip() != "") \
-        #         or (isinstance(m.content, list) and )
-        # ]
-        # 暂时不支持vision，仅保留纯文字的content
         req_messages = []
 
         for m in messages:
             if isinstance(m.content, str) and m.content.strip() != "":
                 req_messages.append(m.dict(exclude_none=True))
             elif isinstance(m.content, list):
-                # 删除m.content中的type!=text的元素
-                m.content = [
-                    c for c in m.content if c.type == "text"
-                ]
+                # m.content = [
+                #     c for c in m.content if c.type == "text"
+                # ]
 
-                if len(m.content) > 0:
-                    req_messages.append(m.dict(exclude_none=True))
+                # if len(m.content) > 0:
+                #     req_messages.append(m.dict(exclude_none=True))
+
+                msg_dict = m.dict(exclude_none=True)
+
+                for i, ce in enumerate(m.content):
+                    if ce.type == "image_url":
+                        alter_image_ele = {
+                            "type": "image",
+                            "source": {
+                                "type": "base64",
+                                "media_type": "image/jpeg",
+                                "data": await image.qq_image_url_to_base64(ce.image_url.url)
+                            }
+                        }
+                        msg_dict["content"][i] = alter_image_ele
+
+                req_messages.append(msg_dict)
 
         args["messages"] = req_messages
 
+        # anthropic的tools处在beta阶段，sdk不稳定，故暂时不支持
+        #
+        # if funcs:
+        #     tools = await self.ap.tool_mgr.generate_tools_for_openai(funcs)
+
+        #     if tools:
+        #         args["tools"] = tools
+
         try:
             resp = await self.client.messages.create(**args)
 

pkg/provider/tools/toolmgr.py

@@ -9,8 +9,7 @@ from ...plugin import context as plugin_context
 
 
 class ToolManager:
-    """LLM工具管理器
-    """
+    """LLM工具管理器"""
 
     ap: app.Application
 
@@ -22,16 +21,16 @@ class ToolManager:
         pass
 
     async def get_function(self, name: str) -> entities.LLMFunction:
-        """获取函数
-        """
+        """获取函数"""
         for function in await self.get_all_functions():
             if function.name == name:
                 return function
         return None
 
-    async def get_function_and_plugin(self, name: str) -> typing.Tuple[entities.LLMFunction, plugin_context.BasePlugin]:
-        """获取函数和插件
-        """
+    async def get_function_and_plugin(
+        self, name: str
+    ) -> typing.Tuple[entities.LLMFunction, plugin_context.BasePlugin]:
+        """获取函数和插件"""
         for plugin in self.ap.plugin_mgr.plugins:
             for function in plugin.content_functions:
                 if function.name == name:
@@ -39,8 +38,7 @@ class ToolManager:
         return None, None
 
     async def get_all_functions(self) -> list[entities.LLMFunction]:
-        """获取所有函数
-        """
+        """获取所有函数"""
         all_functions: list[entities.LLMFunction] = []
 
         for plugin in self.ap.plugin_mgr.plugins:
@@ -48,9 +46,8 @@ class ToolManager:
 
         return all_functions
 
-    async def generate_tools_for_openai(self, use_funcs: entities.LLMFunction) -> str:
-        """生成函数列表
-        """
+    async def generate_tools_for_openai(self, use_funcs: list[entities.LLMFunction]) -> list:
+        """生成函数列表"""
         tools = []
 
         for function in use_funcs:
@@ -60,21 +57,55 @@ class ToolManager:
                     "function": {
                         "name": function.name,
                         "description": function.description,
-                        "parameters": function.parameters
+                        "parameters": function.parameters,
+                    },
                 }
+                tools.append(function_schema)
+
+        return tools
+
+    async def generate_tools_for_anthropic(
+        self, use_funcs: list[entities.LLMFunction]
+    ) -> list:
+        """为anthropic生成函数列表
+
+        e.g.
+
+        [
+          {
+            "name": "get_stock_price",
+            "description": "Get the current stock price for a given ticker symbol.",
+            "input_schema": {
+              "type": "object",
+              "properties": {
+                "ticker": {
+                  "type": "string",
+                  "description": "The stock ticker symbol, e.g. AAPL for Apple Inc."
+                }
+              },
+              "required": ["ticker"]
+            }
+          }
+        ]
+        """
+
+        tools = []
+
+        for function in use_funcs:
+            if function.enable:
+                function_schema = {
+                    "name": function.name,
+                    "description": function.description,
+                    "input_schema": function.parameters,
                 }
                 tools.append(function_schema)
 
         return tools
 
     async def execute_func_call(
-        self,
-        query: core_entities.Query,
-        name: str,
-        parameters: dict
+        self, query: core_entities.Query, name: str, parameters: dict
     ) -> typing.Any:
-        """执行函数调用
-        """
+        """执行函数调用"""
 
         try:
 
@@ -84,16 +115,13 @@ class ToolManager:
 
             parameters = parameters.copy()
 
-            parameters = {
-                "query": query,
-                **parameters
-            }
+            parameters = {"query": query, **parameters}
 
             return await function.func(plugin, **parameters)
         except Exception as e:
-            self.ap.logger.error(f'执行函数 {name} 时发生错误: {e}')
+            self.ap.logger.error(f"执行函数 {name} 时发生错误: {e}")
             traceback.print_exc()
-            return f'error occurred when executing function {name}: {e}'
+            return f"error occurred when executing function {name}: {e}"
         finally:
 
             plugin = None
@@ -107,10 +135,10 @@ class ToolManager:
 
                 await self.ap.ctr_mgr.usage.post_function_record(
                     plugin={
-                        'name': plugin.plugin_name,
-                        'remote': plugin.plugin_source,
-                        'version': plugin.plugin_version,
-                        'author': plugin.plugin_author
+                        "name": plugin.plugin_name,
+                        "remote": plugin.plugin_source,
+                        "version": plugin.plugin_version,
+                        "author": plugin.plugin_author,
                     },
                     function_name=function.name,
                     function_description=function.description,