feat: 模型视觉多模态支持

pkg/pipeline/cntfilter/cntfilter.py

@@ -9,6 +9,7 @@ from ...core import entities as core_entities
 from ...config import manager as cfg_mgr
 from . import filter as filter_model, entities as filter_entities
 from .filters import cntignore, banwords, baiduexamine
+from ...provider import entities as llm_entities
 
 
 @stage.stage_class('PostContentFilterStage')
@@ -141,6 +142,21 @@ class ContentFilterStage(stage.PipelineStage):
         """处理
         """
         if stage_inst_name == 'PreContentFilterStage':
+            
+            contain_non_text = False
+
+            for me in query.message_chain:
+                if not isinstance(me, mirai.Plain):
+                    contain_non_text = True
+                    break
+            
+            if contain_non_text:
+                self.ap.logger.debug(f"消息中包含非文本消息，跳过内容过滤器检查。")
+                return entities.StageProcessResult(
+                    result_type=entities.ResultType.CONTINUE,
+                    new_query=query
+                )
+
             return await self._pre_process(
                 str(query.message_chain).strip(),
                 query

pkg/pipeline/cntfilter/entities.py

@@ -4,6 +4,8 @@ import enum
 
 import pydantic
 
+from ...provider import entities as llm_entities
+
 
 class ResultLevel(enum.Enum):
     """结果等级"""
@@ -29,6 +31,13 @@ class EnableStage(enum.Enum):
     """后处理"""
 
 
+class AcceptContent(enum.Enum):
+    """过滤器接受的内容模态"""
+
+    TEXT = enum.auto()
+
+    IMAGE_URL = enum.auto()
+
 class FilterResult(pydantic.BaseModel):
     level: ResultLevel
     """结果等级
@@ -38,7 +47,7 @@ class FilterResult(pydantic.BaseModel):
     """
 
     replacement: str
-    """替换后的消息
+    """替换后的文本消息
     
     内容过滤器可以进行一些遮掩处理，然后把遮掩后的消息返回。
     若没有修改内容，也需要返回原消息。

pkg/pipeline/cntfilter/filter.py

@@ -5,6 +5,7 @@ import typing
 
 from ...core import app
 from . import entities
+from ...provider import entities as llm_entities
 
 
 preregistered_filters: list[typing.Type[ContentFilter]] = []
@@ -56,6 +57,16 @@ class ContentFilter(metaclass=abc.ABCMeta):
             entities.EnableStage.PRE,
             entities.EnableStage.POST
         ]
+    
+    @property
+    def accept_content(self):
+        """本过滤器接受的模态
+        
+        默认仅接受纯文本
+        """
+        return [
+            entities.AcceptContent.TEXT
+        ]
 
     async def initialize(self):
         """初始化过滤器
@@ -63,7 +74,7 @@ class ContentFilter(metaclass=abc.ABCMeta):
         pass
 
     @abc.abstractmethod
-    async def process(self, message: str) -> entities.FilterResult:
+    async def process(self, message: str=None, image_url=None) -> entities.FilterResult:
         """处理消息
 
         分为前后阶段，具体取决于 enable_stages 的值。
@@ -71,6 +82,7 @@ class ContentFilter(metaclass=abc.ABCMeta):
         
         Args:
             message (str): 需要检查的内容
+            image_url (str): 要检查的图片的 URL
 
         Returns:
             entities.FilterResult: 过滤结果，具体内容请查看 entities.FilterResult 类的文档

pkg/pipeline/preproc/preproc.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import mirai
+
 from .. import stage, entities, stagemgr
 from ...core import entities as core_entities
 from ...provider import entities as llm_entities
@@ -37,9 +39,31 @@ class PreProcessor(stage.PipelineStage):
         query.prompt = conversation.prompt.copy()
         query.messages = conversation.messages.copy()
 
+        # 检查vision是否启用，没启用就删除所有图片
+        if not self.ap.provider_cfg.data['enable-vision']:
+            for msg in query.messages:
+                if isinstance(msg.content, list):
+                    for me in msg.content:
+                        if me.type == 'image_url':
+                            msg.content.remove(me)
+
+        content_list = []
+
+        for me in query.message_chain:
+            if isinstance(me, mirai.Plain):
+                content_list.append(
+                    llm_entities.ContentElement.from_text(me.text)
+                )
+            elif isinstance(me, mirai.Image):
+                if self.ap.provider_cfg.data['enable-vision']:
+                    if me.url is not None:
+                        content_list.append(
+                            llm_entities.ContentElement.from_image_url(str(me.url))
+                        )
+
         query.user_message = llm_entities.Message(  # TODO 适配多模态输入
             role='user',
-            content=str(query.message_chain).strip()
+            content=content_list
         )
 
         query.use_model = conversation.use_model

pkg/pipeline/process/handlers/command.py

@@ -93,15 +93,28 @@ class CommandHandler(handler.MessageHandler):
                         result_type=entities.ResultType.CONTINUE,
                         new_query=query
                     )
-                elif ret.text is not None:
+                elif ret.text is not None or ret.image_url is not None:
+
+                    content: list[llm_entities.ContentElement]= []
+
+                    if ret.text is not None:
+                        content.append(
+                            llm_entities.ContentElement.from_text(ret.text)
+                        )
+
+                    if ret.image_url is not None:
+                        content.append(
+                            llm_entities.ContentElement.from_image_url(ret.image_url)
+                        )
+
                     query.resp_messages.append(
                         llm_entities.Message(
                             role='command',
-                            content=ret.text,
+                            content=content,
                         )
                     )
 
-                    self.ap.logger.info(f'命令返回: {self.cut_str(ret.text)}')
+                    self.ap.logger.info(f'命令返回: {self.cut_str(str(content[0]))}')
 
                     yield entities.StageProcessResult(
                         result_type=entities.ResultType.CONTINUE,

pkg/pipeline/wrapper/wrapper.py

@@ -34,17 +34,19 @@ class ResponseWrapper(stage.PipelineStage):
         """
         
         if query.resp_messages[-1].role == 'command':
-            query.resp_message_chain.append(mirai.MessageChain("[bot] "+query.resp_messages[-1].content))
+            # query.resp_message_chain.append(mirai.MessageChain("[bot] "+query.resp_messages[-1].content))
+            query.resp_message_chain.append(query.resp_messages[-1].get_content_mirai_message_chain(prefix_text='[bot] '))
 
             yield entities.StageProcessResult(
                 result_type=entities.ResultType.CONTINUE,
                 new_query=query
             )
         elif query.resp_messages[-1].role == 'plugin':
-            if not isinstance(query.resp_messages[-1].content, mirai.MessageChain):
-                query.resp_message_chain.append(mirai.MessageChain(query.resp_messages[-1].content))
-            else:
-                query.resp_message_chain.append(query.resp_messages[-1].content)
+            # if not isinstance(query.resp_messages[-1].content, mirai.MessageChain):
+            #     query.resp_message_chain.append(mirai.MessageChain(query.resp_messages[-1].content))
+            # else:
+            #     query.resp_message_chain.append(query.resp_messages[-1].content)
+            query.resp_message_chain.append(query.resp_messages[-1].get_content_mirai_message_chain())
 
             yield entities.StageProcessResult(
                 result_type=entities.ResultType.CONTINUE,
@@ -59,7 +61,7 @@ class ResponseWrapper(stage.PipelineStage):
                 reply_text = ''
 
                 if result.content is not None:  # 有内容
-                    reply_text = result.content
+                    reply_text = str(result.get_content_mirai_message_chain())
 
                     # ============= 触发插件事件 ===============
                     event_ctx = await self.ap.plugin_mgr.emit_event(
@@ -87,7 +89,7 @@ class ResponseWrapper(stage.PipelineStage):
 
                         else:
 
-                            query.resp_message_chain.append(mirai.MessageChain([mirai.Plain(reply_text)]))
+                            query.resp_message_chain.append(result.get_content_mirai_message_chain())
 
                         yield entities.StageProcessResult(
                             result_type=entities.ResultType.CONTINUE,