feat: 模型视觉多模态支持

pkg/provider/entities.py

@@ -21,14 +21,34 @@ class ToolCall(pydantic.BaseModel):
     function: FunctionCall
 
 
-class Content(pydantic.BaseModel):
+class ImageURLContentObject(pydantic.BaseModel):
+    url: str
+
+
+class ContentElement(pydantic.BaseModel):
 
     type: str
     """内容类型"""
 
     text: typing.Optional[str] = None
 
-    image_url: typing.Optional[str] = None
+    image_url: typing.Optional[ImageURLContentObject] = None
+
+    def __str__(self):
+        if self.type == 'text':
+            return self.text
+        elif self.type == 'image_url':
+            return f'[图片]({self.image_url})'
+        else:
+            return '未知内容'
+
+    @classmethod
+    def from_text(cls, text: str):
+        return cls(type='text', text=text)
+
+    @classmethod
+    def from_image_url(cls, image_url: str):
+        return cls(type='image_url', image_url=ImageURLContentObject(url=image_url))
 
 
 class Message(pydantic.BaseModel):
@@ -40,7 +60,7 @@ class Message(pydantic.BaseModel):
     name: typing.Optional[str] = None
     """名称，仅函数调用返回时设置"""
 
-    content: typing.Optional[str] | typing.Optional[mirai.MessageChain] = None
+    content: typing.Optional[list[ContentElement]] | typing.Optional[str] = None
     """内容"""
 
     tool_calls: typing.Optional[list[ToolCall]] = None
@@ -50,8 +70,38 @@ class Message(pydantic.BaseModel):
 
     def readable_str(self) -> str:
         if self.content is not None:
-            return str(self.role) + ": " + str(self.content)
+            return str(self.role) + ": " + str(self.get_content_mirai_message_chain())
         elif self.tool_calls is not None:
             return f'调用工具: {self.tool_calls[0].id}'
         else:
             return '未知消息'
+
+    def get_content_mirai_message_chain(self, prefix_text: str="") -> mirai.MessageChain | None:
+        """将内容转换为 Mirai MessageChain 对象
+        
+        Args:
+            prefix_text (str): 首个文字组件的前缀文本
+        """
+
+        if self.content is None:
+            return None
+        elif isinstance(self.content, str):
+            return mirai.MessageChain([mirai.Plain(prefix_text+self.content)])
+        elif isinstance(self.content, list):
+            mc = []
+            for ce in self.content:
+                if ce.type == 'text':
+                    mc.append(mirai.Plain(ce.text))
+                elif ce.type == 'image':
+                    mc.append(mirai.Image(url=ce.image_url))
+            
+            # 找第一个文字组件
+            if prefix_text:
+                for i, c in enumerate(mc):
+                    if isinstance(c, mirai.Plain):
+                        mc[i] = mirai.Plain(prefix_text+c.text)
+                        break
+                else:
+                    mc.insert(0, mirai.Plain(prefix_text))
+
+            return mirai.MessageChain(mc)

pkg/provider/modelmgr/apis/anthropicmsgs.py

@@ -38,30 +38,42 @@ class AnthropicMessages(api.LLMAPIRequester):
         args = self.ap.provider_cfg.data['requester']['anthropic-messages']['args'].copy()
         args["model"] = model.name if model.model_name is None else model.model_name
 
-        req_messages = [
-            m.dict(exclude_none=True) for m in messages if m.content.strip() != ""
-        ]
+        # 处理消息
 
-        # 删除所有 role=system & content='' 的消息  
-        req_messages = [
-            m for m in req_messages if not (m["role"] == "system" and m["content"].strip() == "")
-        ]
+        # system
+        system_role_message = None
 
-        # 检查是否有 role=system 的消息，若有，改为 role=user，并在后面加一个 role=assistant 的消息
-        system_role_index = []
-        for i, m in enumerate(req_messages):
-            if m["role"] == "system":
-                system_role_index.append(i)
-                m["role"] = "user"
+        for i, m in enumerate(messages):
+            if m.role == "system":
+                system_role_message = m
 
-        if system_role_index:
-            for i in system_role_index[::-1]:
-                req_messages.insert(i + 1, {"role": "assistant", "content": "Okay, I'll follow."})
+                messages.pop(i)
+                break
 
-        # 忽略掉空消息，用户可能发送空消息，而上层未过滤
-        req_messages = [
-            m for m in req_messages if m["content"].strip() != ""
-        ]
+        if isinstance(system_role_message, llm_entities.Message) \
+            and isinstance(system_role_message.content, str):
+            args['system'] = system_role_message.content
+
+        # 其他消息
+        # req_messages = [
+        #     m.dict(exclude_none=True) for m in messages \
+        #         if (isinstance(m.content, str) and m.content.strip() != "") \
+        #         or (isinstance(m.content, list) and )
+        # ]
+        # 暂时不支持vision，仅保留纯文字的content
+        req_messages = []
+
+        for m in messages:
+            if isinstance(m.content, str) and m.content.strip() != "":
+                req_messages.append(m.dict(exclude_none=True))
+            elif isinstance(m.content, list):
+                # 删除m.content中的type!=text的元素
+                m.content = [
+                    c for c in m.content if c.get("type") == "text"
+                ]
+
+                if len(m.content) > 0:
+                    req_messages.append(m.dict(exclude_none=True))
 
         args["messages"] = req_messages
 

pkg/provider/modelmgr/apis/chatcmpl.py

@@ -23,9 +23,18 @@ class OpenAIChatCompletions(api.LLMAPIRequester):
 
     requester_cfg: dict
 
+    cached_image_oss_url: dict[str, str] = {}
+    """缓存的OSS服务的图片URL
+    
+    key: 前文message中的原图片URL（QQ图片）
+    value: OSS服务的图片URL
+    """
+
     def __init__(self, ap: app.Application):
         self.ap = ap
 
+        self.cached_image_oss_url = {}
+
         self.requester_cfg = self.ap.provider_cfg.data['requester']['openai-chat-completions']
 
     async def initialize(self):
@@ -74,7 +83,16 @@ class OpenAIChatCompletions(api.LLMAPIRequester):
                 args["tools"] = tools
 
         # 设置此次请求中的messages
-        messages = req_messages
+        messages = req_messages.copy()
+
+        # 检查vision
+        if self.ap.oss_mgr.available():
+            for msg in messages:
+                if isinstance(msg["content"], list):
+                    for me in msg["content"]:
+                        if me["type"] == "image_url":
+                            me["image_url"]['url'] = await self.get_oss_url(me["image_url"]['url'])
+
         args["messages"] = messages
 
         # 发送请求
@@ -112,3 +130,17 @@ class OpenAIChatCompletions(api.LLMAPIRequester):
             raise errors.RequesterError(f'请求过于频繁或余额不足: {e.message}')
         except openai.APIError as e:
             raise errors.RequesterError(f'请求错误: {e.message}')
+
+    async def get_oss_url(
+        self,
+        original_url: str,
+    ) -> str:
+
+        if original_url in self.cached_image_oss_url:
+            return self.cached_image_oss_url[original_url]
+
+        oss_url = await self.ap.oss_mgr.upload_url_image(original_url)
+
+        self.cached_image_oss_url[original_url] = oss_url
+
+        return oss_url

pkg/provider/modelmgr/apis/deepseekchatcmpl.py

@@ -3,7 +3,10 @@ from __future__ import annotations
 from ....core import app
 
 from . import chatcmpl
-from .. import api
+from .. import api, entities, errors
+from ....core import entities as core_entities, app
+from ... import entities as llm_entities
+from ...tools import entities as tools_entities
 
 
 @api.requester_class("deepseek-chat-completions")
@@ -12,4 +15,39 @@ class DeepseekChatCompletions(chatcmpl.OpenAIChatCompletions):
 
     def __init__(self, ap: app.Application):
         self.requester_cfg = ap.provider_cfg.data['requester']['deepseek-chat-completions']
-        self.ap = ap
+        self.ap = ap
+
+    async def _closure(
+        self,
+        req_messages: list[dict],
+        use_model: entities.LLMModelInfo,
+        use_funcs: list[tools_entities.LLMFunction] = None,
+    ) -> llm_entities.Message:
+        self.client.api_key = use_model.token_mgr.get_token()
+
+        args = self.requester_cfg['args'].copy()
+        args["model"] = use_model.name if use_model.model_name is None else use_model.model_name
+
+        if use_model.tool_call_supported:
+            tools = await self.ap.tool_mgr.generate_tools_for_openai(use_funcs)
+
+            if tools:
+                args["tools"] = tools
+
+        # 设置此次请求中的messages
+        messages = req_messages
+
+        # deepseek 不支持多模态，把content都转换成纯文字
+        for m in messages:
+            if isinstance(m["content"], list):
+                m["content"] = " ".join([c["text"] for c in m["content"]])
+
+        args["messages"] = messages
+
+        # 发送请求
+        resp = await self._req(args)
+
+        # 处理请求结果
+        message = await self._make_msg(resp)
+
+        return message

pkg/provider/modelmgr/apis/moonshotchatcmpl.py

@@ -3,7 +3,10 @@ from __future__ import annotations
 from ....core import app
 
 from . import chatcmpl
-from .. import api
+from .. import api, entities, errors
+from ....core import entities as core_entities, app
+from ... import entities as llm_entities
+from ...tools import entities as tools_entities
 
 
 @api.requester_class("moonshot-chat-completions")
@@ -13,3 +16,41 @@ class MoonshotChatCompletions(chatcmpl.OpenAIChatCompletions):
     def __init__(self, ap: app.Application):
         self.requester_cfg = ap.provider_cfg.data['requester']['moonshot-chat-completions']
         self.ap = ap
+
+    async def _closure(
+        self,
+        req_messages: list[dict],
+        use_model: entities.LLMModelInfo,
+        use_funcs: list[tools_entities.LLMFunction] = None,
+    ) -> llm_entities.Message:
+        self.client.api_key = use_model.token_mgr.get_token()
+
+        args = self.requester_cfg['args'].copy()
+        args["model"] = use_model.name if use_model.model_name is None else use_model.model_name
+
+        if use_model.tool_call_supported:
+            tools = await self.ap.tool_mgr.generate_tools_for_openai(use_funcs)
+
+            if tools:
+                args["tools"] = tools
+
+        # 设置此次请求中的messages
+        messages = req_messages
+
+        # deepseek 不支持多模态，把content都转换成纯文字
+        for m in messages:
+            if isinstance(m["content"], list):
+                m["content"] = " ".join([c["text"] for c in m["content"]])
+
+        # 删除空的
+        messages = [m for m in messages if m["content"].strip() != ""]
+
+        args["messages"] = messages
+
+        # 发送请求
+        resp = await self._req(args)
+
+        # 处理请求结果
+        message = await self._make_msg(resp)
+
+        return message

pkg/provider/modelmgr/modelmgr.py

@@ -37,6 +37,10 @@ class ModelManager:
         raise ValueError(f"无法确定模型 {name} 的信息，请在元数据中配置")
     
     async def initialize(self):
+
+        # 检查是否启用了vision但是没有配置oss
+        if self.ap.provider_cfg.data['enable-vision'] and not self.ap.oss_mgr.available():
+            self.ap.logger.warn("启用了视觉但是没有配置可用的oss服务，基于 URL 传递图片的视觉 API 将无法正常使用")
         
         # 初始化token_mgr, requester
         for k, v in self.ap.provider_cfg.data['keys'].items():