fix:Fixed the issue where the rich text processing in the DingTalk AP… (#1759)

* fix:Fixed the issue where the rich text processing in the DingTalk API did not account for multiple texts and images, as well as the presence of default line breaks. Also resolved the error in Dify caused by sending only images, which resulted in an empty query.

* fix:Considering the various possible scenarios, there are cases where plan_text is empty when there is file content, and there is no file (the message could not be parsed) and the content is empty.

* fix:Add the default modifiable prompt input for didify in the ai.yaml file to ensure that the error of query being empty occurs when receiving data.

* add: The config migration of Dify

* fix:Migration issue

* perf: minor fix

* chore: minor fix

---------

Co-authored-by: Junyan Qin <rockchinq@gmail.com>

pkg/persistence/migrations/dbm011_dify_base_prompt_config.py

@@ -0,0 +1,40 @@
+from .. import migration
+
+import sqlalchemy
+
+from ...entity.persistence import pipeline as persistence_pipeline
+
+
+@migration.migration_class(11)
+class DBMigrateDifyApiConfig(migration.DBMigration):
+    """Langflow API config"""
+
+    async def upgrade(self):
+        """Upgrade"""
+        # read all pipelines
+        pipelines = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
+
+        for pipeline in pipelines:
+            serialized_pipeline = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
+
+            config = serialized_pipeline['config']
+
+            if 'base-prompt' not in config['ai']['dify-service-api']:
+                config['ai']['dify-service-api']['base-prompt'] = (
+                    'When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image.',
+                )
+
+            await self.ap.persistence_mgr.execute_async(
+                sqlalchemy.update(persistence_pipeline.LegacyPipeline)
+                .where(persistence_pipeline.LegacyPipeline.uuid == serialized_pipeline['uuid'])
+                .values(
+                    {
+                        'config': config,
+                        'for_version': self.ap.ver_mgr.get_current_version(),
+                    }
+                )
+            )
+
+    async def downgrade(self):
+        """Downgrade"""
+        pass

pkg/platform/sources/dingtalk.py

@@ -1,3 +1,4 @@
+
 import traceback
 import typing
 from libs.dingtalk_api.dingtalkevent import DingTalkEvent
@@ -36,16 +37,31 @@ class DingTalkMessageConverter(abstract_platform_adapter.AbstractMessageConverte
             if atUser.dingtalk_id == event.incoming_message.chatbot_user_id:
                 yiri_msg_list.append(platform_message.At(target=bot_name))
 
-        if event.content:
-            text_content = event.content.replace('@' + bot_name, '')
-            yiri_msg_list.append(platform_message.Plain(text=text_content))
-        if event.picture:
-            yiri_msg_list.append(platform_message.Image(base64=event.picture))
+        if event.rich_content:
+            elements = event.rich_content.get("Elements")
+            for element in elements:
+                if element.get('Type') == 'text':
+                    text = element.get('Content', '').replace('@' + bot_name, '')
+                    if text.strip():
+                        yiri_msg_list.append(platform_message.Plain(text=text))
+                elif element.get('Type') == 'image' and element.get('Picture'):
+                    yiri_msg_list.append(platform_message.Image(base64=element['Picture']))
+        else:
+            # 回退到原有简单逻辑
+            if event.content:
+                text_content = event.content.replace('@' + bot_name, '')
+                yiri_msg_list.append(platform_message.Plain(text=text_content))
+            if event.picture:
+                yiri_msg_list.append(platform_message.Image(base64=event.picture))
+
+            # 处理其他类型消息（文件、音频等）
         if event.file:
             yiri_msg_list.append(platform_message.File(url=event.file, name=event.name))
         if event.audio:
             yiri_msg_list.append(platform_message.Voice(base64=event.audio))
 
+
+
         chain = platform_message.MessageChain(yiri_msg_list)
 
         return chain

pkg/provider/runners/difysvapi.py

@@ -14,6 +14,7 @@ import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query
 from libs.dify_service_api.v1 import client, errors
 
 
+
 @runner.runner_class('dify-service-api')
 class DifyServiceAPIRunner(runner.RequestRunner):
     """Dify Service API 对话请求器"""
@@ -77,7 +78,7 @@ class DifyServiceAPIRunner(runner.RequestRunner):
             tuple[str, list[str]]: 纯文本和图片的 Dify 服务图片 ID
         """
         plain_text = ''
-        image_ids = []
+        file_ids = []
 
         if isinstance(query.user_message.content, list):
             for ce in query.user_message.content:
@@ -92,11 +93,24 @@ class DifyServiceAPIRunner(runner.RequestRunner):
                         f'{query.session.launcher_type.value}_{query.session.launcher_id}',
                     )
                     image_id = file_upload_resp['id']
-                    image_ids.append(image_id)
+                    file_ids.append(image_id)
+                # elif ce.type == "file_url":
+                #     file_bytes = base64.b64decode(ce.file_url)
+                #     file_upload_resp = await self.dify_client.upload_file(
+                #         file_bytes,
+                #         f'{query.session.launcher_type.value}_{query.session.launcher_id}',
+                #     )
+                #     file_id = file_upload_resp['id']
+                #     file_ids.append(file_id)
         elif isinstance(query.user_message.content, str):
             plain_text = query.user_message.content
+        # plain_text = "When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image." if file_ids and not plain_text else plain_text
+        # plain_text = "The user message type cannot be parsed." if not file_ids and not plain_text else plain_text
+        # plain_text = plain_text if plain_text else "When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image."
+        # print(self.pipeline_config['ai'])
+        plain_text = plain_text if plain_text else self.pipeline_config['ai']['dify-service-api']['base-prompt']
 
-        return plain_text, image_ids
+        return plain_text, file_ids
 
     async def _chat_messages(
         self, query: pipeline_query.Query
@@ -110,7 +124,6 @@ class DifyServiceAPIRunner(runner.RequestRunner):
         files = [
             {
                 'type': 'image',
-                'transfer_method': 'local_file',
                 'upload_file_id': image_id,
             }
             for image_id in image_ids

pkg/utils/constants.py

@@ -1,6 +1,6 @@
 semantic_version = 'v4.4.1'
 
-required_database_version = 10
+required_database_version = 11
 """Tag the version of the database schema, used to check if the database needs to be migrated"""
 
 debug_mode = False