diff --git a/src/langbot/pkg/platform/sources/dingtalk.py b/src/langbot/pkg/platform/sources/dingtalk.py index c072a567..189e3288 100644 --- a/src/langbot/pkg/platform/sources/dingtalk.py +++ b/src/langbot/pkg/platform/sources/dingtalk.py @@ -12,17 +12,41 @@ from langbot.pkg.platform.logger import EventLogger class DingTalkMessageConverter(abstract_platform_adapter.AbstractMessageConverter): @staticmethod - async def yiri2target(message_chain: platform_message.MessageChain): + def _format_image_as_markdown(msg: platform_message.Image) -> str: + """Convert an Image message to Markdown format for DingTalk.""" + if msg.url: + return f'\n![image]({msg.url})\n' + elif msg.base64: + # For base64 images, try to include them as data URIs + # DingTalk may have limited support for base64 in markdown + if msg.base64.startswith('data:'): + return f'\n![image]({msg.base64})\n' + else: + return f'\n![image](data:image/png;base64,{msg.base64})\n' + return '' + + @staticmethod + async def yiri2target(message_chain: platform_message.MessageChain, markdown_enabled: bool = True): content = '' at = False for msg in message_chain: if type(msg) is platform_message.At: at = True - if type(msg) is platform_message.Plain: + elif type(msg) is platform_message.Plain: content += msg.text - if type(msg) is platform_message.Forward: + elif type(msg) is platform_message.Image: + # DingTalk supports markdown images when markdown_card is enabled + # When markdown is disabled, images cannot be rendered in plain text mode + if markdown_enabled: + content += DingTalkMessageConverter._format_image_as_markdown(msg) + # Note: When markdown_enabled is False, images are not included + # as DingTalk plain text messages don't support image embedding + elif type(msg) is platform_message.Forward: for node in msg.node_list: - content += (await DingTalkMessageConverter.yiri2target(node.message_chain))[0] + forwarded_content, _ = await DingTalkMessageConverter.yiri2target( + node.message_chain, markdown_enabled + ) + content += forwarded_content return content, at @staticmethod @@ -157,7 +181,8 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): ) incoming_message = event.incoming_message - content, at = await DingTalkMessageConverter.yiri2target(message) + markdown_enabled = self.config.get('markdown_card', False) + content, at = await DingTalkMessageConverter.yiri2target(message, markdown_enabled) await self.bot.send_message(content, incoming_message, at) async def reply_message_chunk( @@ -178,7 +203,8 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): msg_seq = bot_message.msg_sequence if (msg_seq - 1) % 8 == 0 or is_final: - content, at = await DingTalkMessageConverter.yiri2target(message) + markdown_enabled = self.config.get('markdown_card', False) + content, at = await DingTalkMessageConverter.yiri2target(message, markdown_enabled) card_instance, card_instance_id = self.card_instance_id_dict[message_id] if not content and bot_message.content: @@ -191,7 +217,8 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): self.card_instance_id_dict.pop(message_id) # 消息回复结束之后删除卡片实例id async def send_message(self, target_type: str, target_id: str, message: platform_message.MessageChain): - content = await DingTalkMessageConverter.yiri2target(message) + markdown_enabled = self.config.get('markdown_card', False) + content, _ = await DingTalkMessageConverter.yiri2target(message, markdown_enabled) if target_type == 'person': await self.bot.send_proactive_message_to_one(target_id, content) if target_type == 'group': diff --git a/src/langbot/pkg/platform/sources/lark.py b/src/langbot/pkg/platform/sources/lark.py index 684091a2..e442ed12 100644 --- a/src/langbot/pkg/platform/sources/lark.py +++ b/src/langbot/pkg/platform/sources/lark.py @@ -54,122 +54,179 @@ class AESCipher(object): class LarkMessageConverter(abstract_platform_adapter.AbstractMessageConverter): + @staticmethod + async def upload_image_to_lark( + msg: platform_message.Image, api_client: lark_oapi.Client + ) -> typing.Optional[str]: + """Upload an image to Lark and return the image_key, or None if upload fails.""" + image_bytes = None + + if msg.base64: + try: + # Remove data URL prefix if present + base64_data = msg.base64 + if base64_data.startswith('data:'): + base64_data = base64_data.split(',', 1)[1] + image_bytes = base64.b64decode(base64_data) + except Exception as e: + print(f'Failed to decode base64 image: {e}') + traceback.print_exc() + return None + elif msg.url: + try: + async with aiohttp.ClientSession() as session: + async with session.get(msg.url) as response: + if response.status == 200: + image_bytes = await response.read() + else: + print(f'Failed to download image from {msg.url}: HTTP {response.status}') + return None + except Exception as e: + print(f'Failed to download image from {msg.url}: {e}') + traceback.print_exc() + return None + elif msg.path: + try: + with open(msg.path, 'rb') as f: + image_bytes = f.read() + except Exception as e: + print(f'Failed to read image from path {msg.path}: {e}') + traceback.print_exc() + return None + + if image_bytes is None: + print(f'No image data available for Image message (url={msg.url}, base64={bool(msg.base64)}, path={msg.path})') + return None + + try: + # Create a temporary file to store the image bytes + import tempfile + import os + + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_file.write(image_bytes) + temp_file.flush() + temp_file_path = temp_file.name + + try: + # Create image request using the temporary file + request = ( + CreateImageRequest.builder() + .request_body( + CreateImageRequestBody.builder() + .image_type('message') + .image(open(temp_file_path, 'rb')) + .build() + ) + .build() + ) + + response = await api_client.im.v1.image.acreate(request) + + if not response.success(): + print( + f'client.im.v1.image.create failed, code: {response.code}, msg: {response.msg}, log_id: {response.get_log_id()}' + ) + return None + + return response.data.image_key + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + except Exception as e: + print(f'Failed to upload image to Lark: {e}') + traceback.print_exc() + return None + @staticmethod async def yiri2target( message_chain: platform_message.MessageChain, api_client: lark_oapi.Client - ) -> typing.Tuple[list]: + ) -> typing.Tuple[list, list]: + """Convert message chain to Lark format. + + Returns: + Tuple of (text_elements, image_keys): + - text_elements: List of paragraphs for post message format + - image_keys: List of image_key strings for separate image messages + """ message_elements = [] + image_keys = [] pending_paragraph = [] + + # Regex pattern to match Markdown image syntax: ![alt](url) + markdown_image_pattern = re.compile(r'!\[([^\]]*)\]\(([^)]+)\)') + + async def process_text_with_images(text: str) -> typing.Tuple[str, list]: + """Extract Markdown images from text and return cleaned text + image URLs.""" + extracted_urls = [] + + # Find all Markdown images + matches = list(markdown_image_pattern.finditer(text)) + if not matches: + return text, [] + + # Extract URLs and remove image syntax from text + cleaned_text = text + for match in reversed(matches): # Reverse to maintain correct positions + url = match.group(2) + extracted_urls.insert(0, url) # Insert at beginning since we're going in reverse + # Replace image syntax with empty string or a placeholder + cleaned_text = cleaned_text[:match.start()] + cleaned_text[match.end():] + + # Clean up multiple consecutive newlines that might result from removing images + cleaned_text = re.sub(r'\n{3,}', '\n\n', cleaned_text) + cleaned_text = cleaned_text.strip() + + return cleaned_text, extracted_urls + for msg in message_chain: if isinstance(msg, platform_message.Plain): # Ensure text is valid UTF-8 try: text = msg.text.encode('utf-8').decode('utf-8') - pending_paragraph.append({'tag': 'md', 'text': text}) except UnicodeError: - # If text is not valid UTF-8, try to decode with other encodings try: text = msg.text.encode('latin1').decode('utf-8') - pending_paragraph.append({'tag': 'md', 'text': text}) except UnicodeError: - # If still fails, replace invalid characters text = msg.text.encode('utf-8', errors='replace').decode('utf-8') - pending_paragraph.append({'tag': 'md', 'text': text}) + + # Check for and extract Markdown images from text + cleaned_text, extracted_urls = await process_text_with_images(text) + + # Add cleaned text if not empty + if cleaned_text: + pending_paragraph.append({'tag': 'md', 'text': cleaned_text}) + + # Process extracted image URLs + for url in extracted_urls: + # Create a temporary Image message to upload + temp_image = platform_message.Image(url=url) + image_key = await LarkMessageConverter.upload_image_to_lark(temp_image, api_client) + if image_key: + image_keys.append(image_key) + elif isinstance(msg, platform_message.At): pending_paragraph.append({'tag': 'at', 'user_id': msg.target, 'style': []}) elif isinstance(msg, platform_message.AtAll): pending_paragraph.append({'tag': 'at', 'user_id': 'all', 'style': []}) elif isinstance(msg, platform_message.Image): - image_bytes = None - - if msg.base64: - try: - # Remove data URL prefix if present - if msg.base64.startswith('data:'): - msg.base64 = msg.base64.split(',', 1)[1] - image_bytes = base64.b64decode(msg.base64) - except Exception: - traceback.print_exc() - continue - elif msg.url: - try: - async with aiohttp.ClientSession() as session: - async with session.get(msg.url) as response: - if response.status == 200: - image_bytes = await response.read() - else: - traceback.print_exc() - continue - except Exception: - traceback.print_exc() - continue - elif msg.path: - try: - with open(msg.path, 'rb') as f: - image_bytes = f.read() - except Exception: - traceback.print_exc() - continue - - if image_bytes is None: - continue - - try: - # Create a temporary file to store the image bytes - import tempfile - - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - temp_file.write(image_bytes) - temp_file.flush() - - # Create image request using the temporary file - request = ( - CreateImageRequest.builder() - .request_body( - CreateImageRequestBody.builder() - .image_type('message') - .image(open(temp_file.name, 'rb')) - .build() - ) - .build() - ) - - response = await api_client.im.v1.image.acreate(request) - - if not response.success(): - raise Exception( - f'client.im.v1.image.create failed, code: {response.code}, msg: {response.msg}, log_id: {response.get_log_id()}, resp: \n{json.dumps(json.loads(response.raw.content), indent=4, ensure_ascii=False)}' - ) - - image_key = response.data.image_key - - message_elements.append(pending_paragraph) - message_elements.append( - [ - { - 'tag': 'img', - 'image_key': image_key, - } - ] - ) - pending_paragraph = [] - except Exception: - traceback.print_exc() - continue - finally: - # Clean up the temporary file - import os - - if 'temp_file' in locals(): - os.unlink(temp_file.name) + # Upload image and get image_key + image_key = await LarkMessageConverter.upload_image_to_lark(msg, api_client) + if image_key: + # Store image_key for separate image message + image_keys.append(image_key) elif isinstance(msg, platform_message.Forward): for node in msg.node_list: - message_elements.extend(await LarkMessageConverter.yiri2target(node.message_chain, api_client)) + sub_elements, sub_image_keys = await LarkMessageConverter.yiri2target( + node.message_chain, api_client + ) + message_elements.extend(sub_elements) + image_keys.extend(sub_image_keys) if pending_paragraph: message_elements.append(pending_paragraph) - return message_elements + return message_elements, image_keys @staticmethod async def target2yiri( @@ -667,36 +724,63 @@ class LarkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): ): # 不再需要了，因为message_id已经被包含到message_chain中 # lark_event = await self.event_converter.yiri2target(message_source) - lark_message = await self.message_converter.yiri2target(message, self.api_client) + text_elements, image_keys = await self.message_converter.yiri2target(message, self.api_client) - final_content = { - 'zh_Hans': { - 'title': '', - 'content': lark_message, - }, - } + # Send text message if there are text elements + if text_elements: + final_content = { + 'zh_Hans': { + 'title': '', + 'content': text_elements, + }, + } - request: ReplyMessageRequest = ( - ReplyMessageRequest.builder() - .message_id(message_source.message_chain.message_id) - .request_body( - ReplyMessageRequestBody.builder() - .content(json.dumps(final_content)) - .msg_type('post') - .reply_in_thread(False) - .uuid(str(uuid.uuid4())) + request: ReplyMessageRequest = ( + ReplyMessageRequest.builder() + .message_id(message_source.message_chain.message_id) + .request_body( + ReplyMessageRequestBody.builder() + .content(json.dumps(final_content)) + .msg_type('post') + .reply_in_thread(False) + .uuid(str(uuid.uuid4())) + .build() + ) .build() ) - .build() - ) - response: ReplyMessageResponse = await self.api_client.im.v1.message.areply(request) + response: ReplyMessageResponse = await self.api_client.im.v1.message.areply(request) - if not response.success(): - raise Exception( - f'client.im.v1.message.reply failed, code: {response.code}, msg: {response.msg}, log_id: {response.get_log_id()}, resp: \n{json.dumps(json.loads(response.raw.content), indent=4, ensure_ascii=False)}' + if not response.success(): + raise Exception( + f'client.im.v1.message.reply failed, code: {response.code}, msg: {response.msg}, log_id: {response.get_log_id()}, resp: \n{json.dumps(json.loads(response.raw.content), indent=4, ensure_ascii=False)}' + ) + + # Send image messages separately using msg_type='image' + for image_key in image_keys: + image_content = json.dumps({'image_key': image_key}) + + request: ReplyMessageRequest = ( + ReplyMessageRequest.builder() + .message_id(message_source.message_chain.message_id) + .request_body( + ReplyMessageRequestBody.builder() + .content(image_content) + .msg_type('image') + .reply_in_thread(False) + .uuid(str(uuid.uuid4())) + .build() + ) + .build() ) + response: ReplyMessageResponse = await self.api_client.im.v1.message.areply(request) + + if not response.success(): + raise Exception( + f'client.im.v1.message.reply (image) failed, code: {response.code}, msg: {response.msg}, log_id: {response.get_log_id()}, resp: \n{json.dumps(json.loads(response.raw.content), indent=4, ensure_ascii=False)}' + ) + async def reply_message_chunk( self, message_source: platform_events.MessageEvent, @@ -712,14 +796,15 @@ class LarkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): message_id = bot_message.resp_message_id msg_seq = bot_message.msg_sequence if msg_seq % 8 == 0 or is_final: - lark_message = await self.message_converter.yiri2target(message, self.api_client) + text_elements, image_keys = await self.message_converter.yiri2target(message, self.api_client) text_message = '' - for ele in lark_message[0]: - if ele['tag'] == 'text': - text_message += ele['text'] - elif ele['tag'] == 'md': - text_message += ele['text'] + if text_elements: + for ele in text_elements[0]: + if ele['tag'] == 'text': + text_message += ele['text'] + elif ele['tag'] == 'md': + text_message += ele['text'] # content = { # 'type': 'card_json', diff --git a/src/langbot/pkg/platform/sources/line.py b/src/langbot/pkg/platform/sources/line.py index 29ab361e..413105d6 100644 --- a/src/langbot/pkg/platform/sources/line.py +++ b/src/langbot/pkg/platform/sources/line.py @@ -41,10 +41,9 @@ class LINEMessageConverter(abstract_platform_adapter.AbstractMessageConverter): elif isinstance(component, platform_message.Plain): content_list.append({'type': 'text', 'content': component.text}) elif isinstance(component, platform_message.Image): - if not component.url: - pass - content_list.append({'type': 'image', 'image': component.url}) - + # Only add image if it has a valid URL + if component.url: + content_list.append({'type': 'image', 'image': component.url}) elif isinstance(component, platform_message.Voice): content_list.append({'type': 'voice', 'url': component.url, 'length': component.length}) @@ -207,10 +206,12 @@ class LINEAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): ) ) elif content['type'] == 'image': + # LINE ImageMessage requires original_content_url and preview_image_url + image_url = content['image'] self.bot.reply_message_with_http_info( ReplyMessageRequest( reply_token=message_source.source_platform_object.reply_token, - messages=[ImageMessage(text=content['content'])], + messages=[ImageMessage(original_content_url=image_url, preview_image_url=image_url)], ) ) diff --git a/src/langbot/pkg/platform/sources/slack.py b/src/langbot/pkg/platform/sources/slack.py index dd0ed655..ec6fbae8 100644 --- a/src/langbot/pkg/platform/sources/slack.py +++ b/src/langbot/pkg/platform/sources/slack.py @@ -24,9 +24,20 @@ class SlackMessageConverter(abstract_platform_adapter.AbstractMessageConverter): if type(msg) is platform_message.Plain: content_list.append( { + 'type': 'text', 'content': msg.text, } ) + elif type(msg) is platform_message.Image: + # Slack supports images via unfurling URLs + # Include image URL in the message so Slack can unfurl it + if msg.url: + content_list.append( + { + 'type': 'image', + 'content': msg.url, + } + ) return content_list @@ -116,18 +127,24 @@ class SlackAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): content_list = await SlackMessageConverter.yiri2target(message) for content in content_list: + # Both text and image (URL) are sent as text messages + # Slack will auto-unfurl image URLs + message_content = content['content'] if slack_event.type == 'channel': - await self.bot.send_message_to_channel(content['content'], slack_event.channel_id) + await self.bot.send_message_to_channel(message_content, slack_event.channel_id) if slack_event.type == 'im': - await self.bot.send_message_to_one(content['content'], slack_event.user_id) + await self.bot.send_message_to_one(message_content, slack_event.user_id) async def send_message(self, target_type: str, target_id: str, message: platform_message.MessageChain): content_list = await SlackMessageConverter.yiri2target(message) for content in content_list: + # Both text and image (URL) are sent as text messages + # Slack will auto-unfurl image URLs + message_content = content['content'] if target_type == 'person': - await self.bot.send_message_to_one(content['content'], target_id) + await self.bot.send_message_to_one(message_content, target_id) if target_type == 'group': - await self.bot.send_message_to_channel(content['content'], target_id) + await self.bot.send_message_to_channel(message_content, target_id) def register_listener( self,