fix: command return value image_url handling for DingTalk, Slack, LINE, and Lark adapters (#1810)

* Initial plan

* Fix command return value image_url handling for DingTalk, Slack, and LINE adapters

Co-authored-by: the-lazy-me <52873503+the-lazy-me@users.noreply.github.com>

* Refactor DingTalk image handling into helper method and add clarifying comment

Co-authored-by: the-lazy-me <52873503+the-lazy-me@users.noreply.github.com>

* Fix Lark adapter to not append empty paragraph before images

Co-authored-by: the-lazy-me <52873503+the-lazy-me@users.noreply.github.com>

* Improve Lark adapter image handling with better error logging

Co-authored-by: the-lazy-me <52873503+the-lazy-me@users.noreply.github.com>

* Fix Lark adapter to send images as separate image messages instead of embedded in post

Co-authored-by: the-lazy-me <52873503+the-lazy-me@users.noreply.github.com>

* Parse Markdown image syntax in Lark adapter and render as separate image messages

Co-authored-by: the-lazy-me <52873503+the-lazy-me@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: the-lazy-me <52873503+the-lazy-me@users.noreply.github.com>

src/langbot/pkg/platform/sources/dingtalk.py

@@ -12,17 +12,41 @@ from langbot.pkg.platform.logger import EventLogger
 
 class DingTalkMessageConverter(abstract_platform_adapter.AbstractMessageConverter):
     @staticmethod
-    async def yiri2target(message_chain: platform_message.MessageChain):
+    def _format_image_as_markdown(msg: platform_message.Image) -> str:
+        """Convert an Image message to Markdown format for DingTalk."""
+        if msg.url:
+            return f'\n![image]({msg.url})\n'
+        elif msg.base64:
+            # For base64 images, try to include them as data URIs
+            # DingTalk may have limited support for base64 in markdown
+            if msg.base64.startswith('data:'):
+                return f'\n![image]({msg.base64})\n'
+            else:
+                return f'\n![image](data:image/png;base64,{msg.base64})\n'
+        return ''
+
+    @staticmethod
+    async def yiri2target(message_chain: platform_message.MessageChain, markdown_enabled: bool = True):
         content = ''
         at = False
         for msg in message_chain:
             if type(msg) is platform_message.At:
                 at = True
-            if type(msg) is platform_message.Plain:
+            elif type(msg) is platform_message.Plain:
                 content += msg.text
-            if type(msg) is platform_message.Forward:
+            elif type(msg) is platform_message.Image:
+                # DingTalk supports markdown images when markdown_card is enabled
+                # When markdown is disabled, images cannot be rendered in plain text mode
+                if markdown_enabled:
+                    content += DingTalkMessageConverter._format_image_as_markdown(msg)
+                # Note: When markdown_enabled is False, images are not included
+                # as DingTalk plain text messages don't support image embedding
+            elif type(msg) is platform_message.Forward:
                 for node in msg.node_list:
-                    content += (await DingTalkMessageConverter.yiri2target(node.message_chain))[0]
+                    forwarded_content, _ = await DingTalkMessageConverter.yiri2target(
+                        node.message_chain, markdown_enabled
+                    )
+                    content += forwarded_content
         return content, at
 
     @staticmethod
@@ -157,7 +181,8 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
         )
         incoming_message = event.incoming_message
 
-        content, at = await DingTalkMessageConverter.yiri2target(message)
+        markdown_enabled = self.config.get('markdown_card', False)
+        content, at = await DingTalkMessageConverter.yiri2target(message, markdown_enabled)
         await self.bot.send_message(content, incoming_message, at)
 
     async def reply_message_chunk(
@@ -178,7 +203,8 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
         msg_seq = bot_message.msg_sequence
 
         if (msg_seq - 1) % 8 == 0 or is_final:
-            content, at = await DingTalkMessageConverter.yiri2target(message)
+            markdown_enabled = self.config.get('markdown_card', False)
+            content, at = await DingTalkMessageConverter.yiri2target(message, markdown_enabled)
 
             card_instance, card_instance_id = self.card_instance_id_dict[message_id]
             if not content and bot_message.content:
@@ -191,7 +217,8 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
                 self.card_instance_id_dict.pop(message_id)  # 消息回复结束之后删除卡片实例id
 
     async def send_message(self, target_type: str, target_id: str, message: platform_message.MessageChain):
-        content = await DingTalkMessageConverter.yiri2target(message)
+        markdown_enabled = self.config.get('markdown_card', False)
+        content, _ = await DingTalkMessageConverter.yiri2target(message, markdown_enabled)
         if target_type == 'person':
             await self.bot.send_proactive_message_to_one(target_id, content)
         if target_type == 'group':

src/langbot/pkg/platform/sources/lark.py

@@ -54,122 +54,179 @@ class AESCipher(object):
 
 
 class LarkMessageConverter(abstract_platform_adapter.AbstractMessageConverter):
+    @staticmethod
+    async def upload_image_to_lark(
+        msg: platform_message.Image, api_client: lark_oapi.Client
+    ) -> typing.Optional[str]:
+        """Upload an image to Lark and return the image_key, or None if upload fails."""
+        image_bytes = None
+
+        if msg.base64:
+            try:
+                # Remove data URL prefix if present
+                base64_data = msg.base64
+                if base64_data.startswith('data:'):
+                    base64_data = base64_data.split(',', 1)[1]
+                image_bytes = base64.b64decode(base64_data)
+            except Exception as e:
+                print(f'Failed to decode base64 image: {e}')
+                traceback.print_exc()
+                return None
+        elif msg.url:
+            try:
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(msg.url) as response:
+                        if response.status == 200:
+                            image_bytes = await response.read()
+                        else:
+                            print(f'Failed to download image from {msg.url}: HTTP {response.status}')
+                            return None
+            except Exception as e:
+                print(f'Failed to download image from {msg.url}: {e}')
+                traceback.print_exc()
+                return None
+        elif msg.path:
+            try:
+                with open(msg.path, 'rb') as f:
+                    image_bytes = f.read()
+            except Exception as e:
+                print(f'Failed to read image from path {msg.path}: {e}')
+                traceback.print_exc()
+                return None
+
+        if image_bytes is None:
+            print(f'No image data available for Image message (url={msg.url}, base64={bool(msg.base64)}, path={msg.path})')
+            return None
+
+        try:
+            # Create a temporary file to store the image bytes
+            import tempfile
+            import os
+
+            with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+                temp_file.write(image_bytes)
+                temp_file.flush()
+                temp_file_path = temp_file.name
+
+            try:
+                # Create image request using the temporary file
+                request = (
+                    CreateImageRequest.builder()
+                    .request_body(
+                        CreateImageRequestBody.builder()
+                        .image_type('message')
+                        .image(open(temp_file_path, 'rb'))
+                        .build()
+                    )
+                    .build()
+                )
+
+                response = await api_client.im.v1.image.acreate(request)
+
+                if not response.success():
+                    print(
+                        f'client.im.v1.image.create failed, code: {response.code}, msg: {response.msg}, log_id: {response.get_log_id()}'
+                    )
+                    return None
+
+                return response.data.image_key
+            finally:
+                # Clean up the temporary file
+                os.unlink(temp_file_path)
+        except Exception as e:
+            print(f'Failed to upload image to Lark: {e}')
+            traceback.print_exc()
+            return None
+
     @staticmethod
     async def yiri2target(
         message_chain: platform_message.MessageChain, api_client: lark_oapi.Client
-    ) -> typing.Tuple[list]:
+    ) -> typing.Tuple[list, list]:
+        """Convert message chain to Lark format.
+        
+        Returns:
+            Tuple of (text_elements, image_keys):
+            - text_elements: List of paragraphs for post message format
+            - image_keys: List of image_key strings for separate image messages
+        """
         message_elements = []
+        image_keys = []
         pending_paragraph = []
+
+        # Regex pattern to match Markdown image syntax: ![alt](url)
+        markdown_image_pattern = re.compile(r'!\[([^\]]*)\]\(([^)]+)\)')
+
+        async def process_text_with_images(text: str) -> typing.Tuple[str, list]:
+            """Extract Markdown images from text and return cleaned text + image URLs."""
+            extracted_urls = []
+            
+            # Find all Markdown images
+            matches = list(markdown_image_pattern.finditer(text))
+            if not matches:
+                return text, []
+            
+            # Extract URLs and remove image syntax from text
+            cleaned_text = text
+            for match in reversed(matches):  # Reverse to maintain correct positions
+                url = match.group(2)
+                extracted_urls.insert(0, url)  # Insert at beginning since we're going in reverse
+                # Replace image syntax with empty string or a placeholder
+                cleaned_text = cleaned_text[:match.start()] + cleaned_text[match.end():]
+            
+            # Clean up multiple consecutive newlines that might result from removing images
+            cleaned_text = re.sub(r'\n{3,}', '\n\n', cleaned_text)
+            cleaned_text = cleaned_text.strip()
+            
+            return cleaned_text, extracted_urls
+
         for msg in message_chain:
             if isinstance(msg, platform_message.Plain):
                 # Ensure text is valid UTF-8
                 try:
                     text = msg.text.encode('utf-8').decode('utf-8')
-                    pending_paragraph.append({'tag': 'md', 'text': text})
                 except UnicodeError:
-                    # If text is not valid UTF-8, try to decode with other encodings
                     try:
                         text = msg.text.encode('latin1').decode('utf-8')
-                        pending_paragraph.append({'tag': 'md', 'text': text})
                     except UnicodeError:
-                        # If still fails, replace invalid characters
                         text = msg.text.encode('utf-8', errors='replace').decode('utf-8')
-                        pending_paragraph.append({'tag': 'md', 'text': text})
+                
+                # Check for and extract Markdown images from text
+                cleaned_text, extracted_urls = await process_text_with_images(text)
+                
+                # Add cleaned text if not empty
+                if cleaned_text:
+                    pending_paragraph.append({'tag': 'md', 'text': cleaned_text})
+                
+                # Process extracted image URLs
+                for url in extracted_urls:
+                    # Create a temporary Image message to upload
+                    temp_image = platform_message.Image(url=url)
+                    image_key = await LarkMessageConverter.upload_image_to_lark(temp_image, api_client)
+                    if image_key:
+                        image_keys.append(image_key)
+                        
             elif isinstance(msg, platform_message.At):
                 pending_paragraph.append({'tag': 'at', 'user_id': msg.target, 'style': []})
             elif isinstance(msg, platform_message.AtAll):
                 pending_paragraph.append({'tag': 'at', 'user_id': 'all', 'style': []})
             elif isinstance(msg, platform_message.Image):
-                image_bytes = None
+                # Upload image and get image_key
-
+                image_key = await LarkMessageConverter.upload_image_to_lark(msg, api_client)
-                if msg.base64:
+                if image_key:
-                    try:
+                    # Store image_key for separate image message
-                        # Remove data URL prefix if present
+                    image_keys.append(image_key)
-                        if msg.base64.startswith('data:'):
-                            msg.base64 = msg.base64.split(',', 1)[1]
-                        image_bytes = base64.b64decode(msg.base64)
-                    except Exception:
-                        traceback.print_exc()
-                        continue
-                elif msg.url:
-                    try:
-                        async with aiohttp.ClientSession() as session:
-                            async with session.get(msg.url) as response:
-                                if response.status == 200:
-                                    image_bytes = await response.read()
-                                else:
-                                    traceback.print_exc()
-                                    continue
-                    except Exception:
-                        traceback.print_exc()
-                        continue
-                elif msg.path:
-                    try:
-                        with open(msg.path, 'rb') as f:
-                            image_bytes = f.read()
-                    except Exception:
-                        traceback.print_exc()
-                        continue
-
-                if image_bytes is None:
-                    continue
-
-                try:
-                    # Create a temporary file to store the image bytes
-                    import tempfile
-
-                    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-                        temp_file.write(image_bytes)
-                        temp_file.flush()
-
-                        # Create image request using the temporary file
-                        request = (
-                            CreateImageRequest.builder()
-                            .request_body(
-                                CreateImageRequestBody.builder()
-                                .image_type('message')
-                                .image(open(temp_file.name, 'rb'))
-                                .build()
-                            )
-                            .build()
-                        )
-
-                        response = await api_client.im.v1.image.acreate(request)
-
-                        if not response.success():
-                            raise Exception(
-                                f'client.im.v1.image.create failed, code: {response.code}, msg: {response.msg}, log_id: {response.get_log_id()}, resp: \n{json.dumps(json.loads(response.raw.content), indent=4, ensure_ascii=False)}'
-                            )
-
-                        image_key = response.data.image_key
-
-                        message_elements.append(pending_paragraph)
-                        message_elements.append(
-                            [
-                                {
-                                    'tag': 'img',
-                                    'image_key': image_key,
-                                }
-                            ]
-                        )
-                        pending_paragraph = []
-                except Exception:
-                    traceback.print_exc()
-                    continue
-                finally:
-                    # Clean up the temporary file
-                    import os
-
-                    if 'temp_file' in locals():
-                        os.unlink(temp_file.name)
             elif isinstance(msg, platform_message.Forward):
                 for node in msg.node_list:
-                    message_elements.extend(await LarkMessageConverter.yiri2target(node.message_chain, api_client))
+                    sub_elements, sub_image_keys = await LarkMessageConverter.yiri2target(
+                        node.message_chain, api_client
+                    )
+                    message_elements.extend(sub_elements)
+                    image_keys.extend(sub_image_keys)
 
         if pending_paragraph:
             message_elements.append(pending_paragraph)
 
-        return message_elements
+        return message_elements, image_keys
 
     @staticmethod
     async def target2yiri(
@@ -667,36 +724,63 @@ class LarkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
     ):
         # 不再需要了，因为message_id已经被包含到message_chain中
         # lark_event = await self.event_converter.yiri2target(message_source)
-        lark_message = await self.message_converter.yiri2target(message, self.api_client)
+        text_elements, image_keys = await self.message_converter.yiri2target(message, self.api_client)
 
-        final_content = {
+        # Send text message if there are text elements
-            'zh_Hans': {
+        if text_elements:
-                'title': '',
+            final_content = {
-                'content': lark_message,
+                'zh_Hans': {
-            },
+                    'title': '',
-        }
+                    'content': text_elements,
+                },
+            }
 
-        request: ReplyMessageRequest = (
+            request: ReplyMessageRequest = (
-            ReplyMessageRequest.builder()
+                ReplyMessageRequest.builder()
-            .message_id(message_source.message_chain.message_id)
+                .message_id(message_source.message_chain.message_id)
-            .request_body(
+                .request_body(
-                ReplyMessageRequestBody.builder()
+                    ReplyMessageRequestBody.builder()
-                .content(json.dumps(final_content))
+                    .content(json.dumps(final_content))
-                .msg_type('post')
+                    .msg_type('post')
-                .reply_in_thread(False)
+                    .reply_in_thread(False)
-                .uuid(str(uuid.uuid4()))
+                    .uuid(str(uuid.uuid4()))
+                    .build()
+                )
                 .build()
             )
-            .build()
-        )
 
-        response: ReplyMessageResponse = await self.api_client.im.v1.message.areply(request)
+            response: ReplyMessageResponse = await self.api_client.im.v1.message.areply(request)
 
-        if not response.success():
+            if not response.success():
-            raise Exception(
+                raise Exception(
-                f'client.im.v1.message.reply failed, code: {response.code}, msg: {response.msg}, log_id: {response.get_log_id()}, resp: \n{json.dumps(json.loads(response.raw.content), indent=4, ensure_ascii=False)}'
+                    f'client.im.v1.message.reply failed, code: {response.code}, msg: {response.msg}, log_id: {response.get_log_id()}, resp: \n{json.dumps(json.loads(response.raw.content), indent=4, ensure_ascii=False)}'
+                )
+
+        # Send image messages separately using msg_type='image'
+        for image_key in image_keys:
+            image_content = json.dumps({'image_key': image_key})
+
+            request: ReplyMessageRequest = (
+                ReplyMessageRequest.builder()
+                .message_id(message_source.message_chain.message_id)
+                .request_body(
+                    ReplyMessageRequestBody.builder()
+                    .content(image_content)
+                    .msg_type('image')
+                    .reply_in_thread(False)
+                    .uuid(str(uuid.uuid4()))
+                    .build()
+                )
+                .build()
             )
 
+            response: ReplyMessageResponse = await self.api_client.im.v1.message.areply(request)
+
+            if not response.success():
+                raise Exception(
+                    f'client.im.v1.message.reply (image) failed, code: {response.code}, msg: {response.msg}, log_id: {response.get_log_id()}, resp: \n{json.dumps(json.loads(response.raw.content), indent=4, ensure_ascii=False)}'
+                )
+
     async def reply_message_chunk(
         self,
         message_source: platform_events.MessageEvent,
@@ -712,14 +796,15 @@ class LarkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
         message_id = bot_message.resp_message_id
         msg_seq = bot_message.msg_sequence
         if msg_seq % 8 == 0 or is_final:
-            lark_message = await self.message_converter.yiri2target(message, self.api_client)
+            text_elements, image_keys = await self.message_converter.yiri2target(message, self.api_client)
 
             text_message = ''
-            for ele in lark_message[0]:
+            if text_elements:
-                if ele['tag'] == 'text':
+                for ele in text_elements[0]:
-                    text_message += ele['text']
+                    if ele['tag'] == 'text':
-                elif ele['tag'] == 'md':
+                        text_message += ele['text']
-                    text_message += ele['text']
+                    elif ele['tag'] == 'md':
+                        text_message += ele['text']
 
             # content = {
             #     'type': 'card_json',

src/langbot/pkg/platform/sources/line.py

@@ -41,10 +41,9 @@ class LINEMessageConverter(abstract_platform_adapter.AbstractMessageConverter):
             elif isinstance(component, platform_message.Plain):
                 content_list.append({'type': 'text', 'content': component.text})
             elif isinstance(component, platform_message.Image):
-                if not component.url:
+                # Only add image if it has a valid URL
-                    pass
+                if component.url:
-                content_list.append({'type': 'image', 'image': component.url})
+                    content_list.append({'type': 'image', 'image': component.url})
-
             elif isinstance(component, platform_message.Voice):
                 content_list.append({'type': 'voice', 'url': component.url, 'length': component.length})
 
@@ -207,10 +206,12 @@ class LINEAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
                     )
                 )
             elif content['type'] == 'image':
+                # LINE ImageMessage requires original_content_url and preview_image_url
+                image_url = content['image']
                 self.bot.reply_message_with_http_info(
                     ReplyMessageRequest(
                         reply_token=message_source.source_platform_object.reply_token,
-                        messages=[ImageMessage(text=content['content'])],
+                        messages=[ImageMessage(original_content_url=image_url, preview_image_url=image_url)],
                     )
                 )
 

src/langbot/pkg/platform/sources/slack.py

@@ -24,9 +24,20 @@ class SlackMessageConverter(abstract_platform_adapter.AbstractMessageConverter):
             if type(msg) is platform_message.Plain:
                 content_list.append(
                     {
+                        'type': 'text',
                         'content': msg.text,
                     }
                 )
+            elif type(msg) is platform_message.Image:
+                # Slack supports images via unfurling URLs
+                # Include image URL in the message so Slack can unfurl it
+                if msg.url:
+                    content_list.append(
+                        {
+                            'type': 'image',
+                            'content': msg.url,
+                        }
+                    )
 
         return content_list
 
@@ -116,18 +127,24 @@ class SlackAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
         content_list = await SlackMessageConverter.yiri2target(message)
 
         for content in content_list:
+            # Both text and image (URL) are sent as text messages
+            # Slack will auto-unfurl image URLs
+            message_content = content['content']
             if slack_event.type == 'channel':
-                await self.bot.send_message_to_channel(content['content'], slack_event.channel_id)
+                await self.bot.send_message_to_channel(message_content, slack_event.channel_id)
             if slack_event.type == 'im':
-                await self.bot.send_message_to_one(content['content'], slack_event.user_id)
+                await self.bot.send_message_to_one(message_content, slack_event.user_id)
 
     async def send_message(self, target_type: str, target_id: str, message: platform_message.MessageChain):
         content_list = await SlackMessageConverter.yiri2target(message)
         for content in content_list:
+            # Both text and image (URL) are sent as text messages
+            # Slack will auto-unfurl image URLs
+            message_content = content['content']
             if target_type == 'person':
-                await self.bot.send_message_to_one(content['content'], target_id)
+                await self.bot.send_message_to_one(message_content, target_id)
             if target_type == 'group':
-                await self.bot.send_message_to_channel(content['content'], target_id)
+                await self.bot.send_message_to_channel(message_content, target_id)
 
     def register_listener(
         self,