feat: 模型视觉多模态支持

pkg/pipeline/cntfilter/cntfilter.py

@@ -9,6 +9,7 @@ from ...core import entities as core_entities
 from ...config import manager as cfg_mgr
 from . import filter as filter_model, entities as filter_entities
 from .filters import cntignore, banwords, baiduexamine
+from ...provider import entities as llm_entities
 
 
 @stage.stage_class('PostContentFilterStage')
@@ -141,6 +142,21 @@ class ContentFilterStage(stage.PipelineStage):
         """处理
         """
         if stage_inst_name == 'PreContentFilterStage':
+            
+            contain_non_text = False
+
+            for me in query.message_chain:
+                if not isinstance(me, mirai.Plain):
+                    contain_non_text = True
+                    break
+            
+            if contain_non_text:
+                self.ap.logger.debug(f"消息中包含非文本消息，跳过内容过滤器检查。")
+                return entities.StageProcessResult(
+                    result_type=entities.ResultType.CONTINUE,
+                    new_query=query
+                )
+
             return await self._pre_process(
                 str(query.message_chain).strip(),
                 query

pkg/pipeline/cntfilter/entities.py

@@ -4,6 +4,8 @@ import enum
 
 import pydantic
 
+from ...provider import entities as llm_entities
+
 
 class ResultLevel(enum.Enum):
     """结果等级"""
@@ -29,6 +31,13 @@ class EnableStage(enum.Enum):
     """后处理"""
 
 
+class AcceptContent(enum.Enum):
+    """过滤器接受的内容模态"""
+
+    TEXT = enum.auto()
+
+    IMAGE_URL = enum.auto()
+
 class FilterResult(pydantic.BaseModel):
     level: ResultLevel
     """结果等级
@@ -38,7 +47,7 @@ class FilterResult(pydantic.BaseModel):
     """
 
     replacement: str
-    """替换后的消息
+    """替换后的文本消息
     
     内容过滤器可以进行一些遮掩处理，然后把遮掩后的消息返回。
     若没有修改内容，也需要返回原消息。

pkg/pipeline/cntfilter/filter.py

@@ -5,6 +5,7 @@ import typing
 
 from ...core import app
 from . import entities
+from ...provider import entities as llm_entities
 
 
 preregistered_filters: list[typing.Type[ContentFilter]] = []
@@ -56,6 +57,16 @@ class ContentFilter(metaclass=abc.ABCMeta):
             entities.EnableStage.PRE,
             entities.EnableStage.POST
         ]
+    
+    @property
+    def accept_content(self):
+        """本过滤器接受的模态
+        
+        默认仅接受纯文本
+        """
+        return [
+            entities.AcceptContent.TEXT
+        ]
 
     async def initialize(self):
         """初始化过滤器
@@ -63,7 +74,7 @@ class ContentFilter(metaclass=abc.ABCMeta):
         pass
 
     @abc.abstractmethod
-    async def process(self, message: str) -> entities.FilterResult:
+    async def process(self, message: str=None, image_url=None) -> entities.FilterResult:
         """处理消息
 
         分为前后阶段，具体取决于 enable_stages 的值。
@@ -71,6 +82,7 @@ class ContentFilter(metaclass=abc.ABCMeta):
         
         Args:
             message (str): 需要检查的内容
+            image_url (str): 要检查的图片的 URL
 
         Returns:
             entities.FilterResult: 过滤结果，具体内容请查看 entities.FilterResult 类的文档