fix(wecombot): deliver sandbox outbox media through full pipeline chain (#2328)

* fix(wecombot): align media upload protocol * fix(wecombot): deliver outbox media in reply and fix tool call recording - Integrate _send_media into reply_message and reply_message_chunk so sandbox outbox images/voices/files are uploaded and sent instead of being silently dropped. - Add missing import base64 that caused _send_media to fail with a NameError swallowed by its except clause. - Change yiri2target to return component dicts (text/image/voice/file) so callers can distinguish text from media. - Fix _get_message_for_tool_context using result.first()/row[0] which returned a raw string instead of the ORM object, causing "'str' object has no attribute 'pipeline_id'" in tool call recording. Use result.scalars().first() per SQLAlchemy 2.0 convention. * fix(pipeline): collect outbox attachments on final chunk with empty content When the last streaming chunk has is_final=True but empty content (e.g. the LLM sends all text in earlier chunks), the 'if result.content' branch is skipped entirely, so _append_outbound_attachments never runs and sandbox outbox images are silently dropped. Add an elif branch for _is_final_assistant_message that creates an empty MessageChain and still collects outbox attachments, so images are delivered even when the final chunk carries no text. * fix(box): bypass stdout truncation when reading outbox via exec _read_outbox_via_exec used execute_tool which returns _serialize_result where stdout is truncated to output_limit_chars (4000). A 7KB JPEG encodes to ~9400 base64 chars, so the JSON payload was truncated and json.loads failed silently, returning an empty list. Call client.execute directly to get the raw BoxExecutionResult with untruncated stdout, so base64 file data is preserved. * fix(tests): adapt box and wrapper tests for client.execute and strict is_final check - wrapper.py: restrict outbox collection on empty-content chunks to actual MessageChunk instances with is_final=True, not generic Mock objects that happen to have role='assistant' - test_box_service.py: update _read_outbox_via_exec tests to mock client.execute (returning BoxExecutionResult) instead of execute_tool, matching the implementation change * chore(wecombot): remove temporary upload log * test(box): preserve direct outbox read and cleanup coverage --------- Co-authored-by: fdc310 <2213070223@qq.com> Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com>
2026-08-22 10:17:13 +00:00 · 2026-08-20 16:56:20 +08:00
parent 7803d56254
commit e934f08adf
6 changed files with 458 additions and 45 deletions
@@ -46,6 +46,14 @@ CMD_RESPOND_MSG = 'aibot_respond_msg'
 CMD_RESPOND_WELCOME = 'aibot_respond_welcome_msg'
 CMD_RESPOND_UPDATE = 'aibot_respond_update_msg'
 CMD_SEND_MSG = 'aibot_send_msg'
+# Media upload protocol (3 steps: init -> chunk * N -> finish). The
+# command names below match the WeCom AI Bot long-connection protocol.
+CMD_UPLOAD_INIT = 'aibot_upload_media_init'
+CMD_UPLOAD_CHUNK = 'aibot_upload_media_chunk'
+CMD_UPLOAD_FINISH = 'aibot_upload_media_finish'
+
+# Default upload chunk size: 512 KB before base64 encoding.
+_UPLOAD_CHUNK_SIZE = 512 * 1024

 _DEDUP_CACHE_MAX = 4096
 _STREAM_CACHE_MAX = 1024
@@ -495,6 +503,145 @@ class WecomBotWsClient:
        body['chatid'] = chat_id
        return await self._send_reply(req_id, body, cmd=CMD_SEND_MSG)

+    # ------------------------------------------------------------------
+    # Media upload (image / voice / file)
+    # ------------------------------------------------------------------
+
+    async def upload_media(
+        self,
+        data: bytes,
+        filename: str = 'attachment',
+        media_type: str = 'file',
+    ) -> Optional[dict]:
+        """Upload *data* to the WeCom AI Bot CDN and return the parsed ACK.
+
+        Implements the three-step protocol documented for the WeCom
+        AI Bot:
+
+        1. ``aibot_upload_media_init`` — declare media type, file name,
+           size, MD5 and chunk count; receive ``upload_id``.
+        2. ``aibot_upload_media_chunk`` — send each chunk (base64-encoded
+           bytes) until done; receive per-chunk ACK.
+        3. ``aibot_upload_media_finish`` — finalize the upload; receive
+           ``media_id``.
+
+        Returns a dict with the final ``media_id`` (and the raw
+        ``finish`` ACK) on success, or ``None`` on any failure. The
+        caller is expected to ignore the result and continue
+        gracefully — the framework will keep working without media
+        delivery.
+        """
+        import base64 as _b64
+        import hashlib as _hl
+
+        if not data:
+            return None
+
+        file_size = len(data)
+        file_md5 = _hl.md5(data).hexdigest()
+        total_chunks = (file_size + _UPLOAD_CHUNK_SIZE - 1) // _UPLOAD_CHUNK_SIZE
+        if total_chunks == 0:
+            total_chunks = 1
+
+        # Step 1: init.
+        init_req_id = _generate_req_id(CMD_UPLOAD_INIT)
+        init_body = {
+            'type': media_type,
+            'filename': filename,
+            'total_size': file_size,
+            'total_chunks': total_chunks,
+            'md5': file_md5,
+        }
+        init_ack = await self._send_reply(
+            init_req_id,
+            init_body,
+            cmd=CMD_UPLOAD_INIT,
+        )
+        if not init_ack or init_ack.get('errcode', 0) != 0:
+            await self.logger.warning(f'upload_media init failed: ack={init_ack!r}')
+            return None
+        upload_id = (
+            init_ack.get('upload_id')
+            or init_ack.get('body', {}).get('upload_id')
+            or init_ack.get('data', {}).get('upload_id')
+        )
+        if not upload_id:
+            await self.logger.warning(f'upload_media init returned no upload_id: ack={init_ack!r}')
+            return None
+
+        # Step 2: chunks.
+        for index in range(total_chunks):
+            start = index * _UPLOAD_CHUNK_SIZE
+            end = min(start + _UPLOAD_CHUNK_SIZE, file_size)
+            chunk_bytes = data[start:end]
+            chunk_req_id = _generate_req_id(CMD_UPLOAD_CHUNK)
+            chunk_body = {
+                'upload_id': upload_id,
+                'chunk_index': index,
+                'base64_data': _b64.b64encode(chunk_bytes).decode('ascii'),
+            }
+            chunk_ack = await self._send_reply(
+                chunk_req_id,
+                chunk_body,
+                cmd=CMD_UPLOAD_CHUNK,
+            )
+            if not chunk_ack or chunk_ack.get('errcode', 0) != 0:
+                await self.logger.warning(f'upload_media chunk {index} failed: ack={chunk_ack!r}')
+                return None
+
+        # Step 3: finish.
+        finish_req_id = _generate_req_id(CMD_UPLOAD_FINISH)
+        finish_body = {'upload_id': upload_id}
+        finish_ack = await self._send_reply(
+            finish_req_id,
+            finish_body,
+            cmd=CMD_UPLOAD_FINISH,
+        )
+        if not finish_ack or finish_ack.get('errcode', 0) != 0:
+            await self.logger.warning(f'upload_media finish failed: ack={finish_ack!r}')
+            return None
+
+        media_id = (
+            finish_ack.get('media_id')
+            or finish_ack.get('body', {}).get('media_id')
+            or finish_ack.get('data', {}).get('media_id')
+        )
+        if not media_id:
+            await self.logger.warning(f'upload_media finish returned no media_id: ack={finish_ack!r}')
+            return None
+        return {'media_id': media_id, 'ack': finish_ack}
+
+    async def _reply_media(
+        self,
+        req_id: str,
+        media_id: str,
+        kind: str,
+    ) -> Optional[dict]:
+        """Send a media reply (image / voice / file) referencing *media_id*.
+
+        ``kind`` is one of ``'image'``, ``'voice'``, ``'file'``. Uses
+        the standard ``aibot_respond_msg`` command with a per-kind
+        body key (matches the convention documented for the WeCom
+        AI Bot SDK).
+        """
+        if kind not in {'image', 'voice', 'file'}:
+            await self.logger.warning(f'_reply_media called with unknown kind={kind!r}')
+            return None
+        body = {
+            'msgtype': kind,
+            kind: {'media_id': media_id},
+        }
+        return await self._send_reply(req_id, body, cmd=CMD_RESPOND_MSG)
+
+    async def reply_image(self, req_id: str, media_id: str) -> Optional[dict]:
+        return await self._reply_media(req_id, media_id, 'image')
+
+    async def reply_file(self, req_id: str, media_id: str) -> Optional[dict]:
+        return await self._reply_media(req_id, media_id, 'file')
+
+    async def reply_voice(self, req_id: str, media_id: str) -> Optional[dict]:
+        return await self._reply_media(req_id, media_id, 'voice')
+
    async def push_stream_chunk(self, msg_id: str, content: str, is_final: bool = False) -> bool:
        """Push a streaming chunk for a given message ID.

@@ -1210,8 +1210,9 @@ class BoxService:
    async def _read_outbox_via_exec(self, query: pipeline_query.Query) -> list[dict]:
        """Fallback: read the outbox over the exec channel (E2B / remote).

-        Note: exec stdout is truncated by ``output_limit_chars``, so this path
-        only reliably transfers small files. The host path is preferred.
+        Uses ``client.execute`` directly (bypassing ``_serialize_result``)
+        so stdout is NOT truncated by ``output_limit_chars`` - the raw
+        base64 payload can be far larger than the 4000-char display limit.
        """
        import json as _json

@@ -1265,14 +1266,22 @@ class BoxService:
            '                break\n'
            'print(json.dumps(out))\n'
        )
-        result = await self.execute_tool(
-            {'command': f"python3 - <<'LBPY'\n{script}\nLBPY", 'timeout_sec': 120},
-            query,
-        )
-        if not result.get('ok'):
+        spec_payload: dict = {
+            'cmd': f"python3 - <<'LBPY'\n{script}\nLBPY",
+            'timeout_sec': 120,
+            'session_id': self.resolve_box_session_id(query),
+        }
+        if 'extra_mounts' not in spec_payload:
+            spec_payload['extra_mounts'] = self.build_skill_extra_mounts(query)
+        try:
+            spec = self.build_spec(spec_payload)
+            result = await self.client.execute(spec)
+        except Exception:
+            return []
+        if not result.ok:
            return []
        try:
-            return _json.loads(str(result.get('stdout') or '').strip().splitlines()[-1])
+            return _json.loads(str(result.stdout or '').strip().splitlines()[-1])
        except Exception:
            return []

@@ -158,6 +158,18 @@ class ResponseWrapper(stage.PipelineStage):
                                result_type=entities.ResultType.CONTINUE,
                                new_query=query,
                            )
+                    elif (
+                        isinstance(result, provider_message.MessageChunk) and result.is_final and not result.tool_calls
+                    ):
+                        # Final streaming chunk with no text content but
+                        # possibly carrying sandbox outbox attachments.
+                        reply_chain = platform_message.MessageChain([])
+                        await self._append_outbound_attachments(query, reply_chain)
+                        query.resp_message_chain.append(reply_chain)
+                        yield entities.StageProcessResult(
+                            result_type=entities.ResultType.CONTINUE,
+                            new_query=query,
+                        )

                    if result.tool_calls is not None and len(result.tool_calls) > 0:  # 有函数调用
                        function_names = [tc.function.name for tc in result.tool_calls]
@@ -3,8 +3,10 @@ import typing
 import asyncio
 import time
 import traceback
+import base64

 import datetime
+
 import langbot_plugin.api.definition.abstract.platform.adapter as abstract_platform_adapter
 import langbot_plugin.api.entities.builtin.platform.message as platform_message
 import langbot_plugin.api.entities.builtin.platform.events as platform_events
@@ -24,11 +26,24 @@ from langbot.libs.wecom_ai_bot_api.ws_client import WecomBotWsClient
 class WecomBotMessageConverter(abstract_platform_adapter.AbstractMessageConverter):
    @staticmethod
    async def yiri2target(message_chain: platform_message.MessageChain):
-        content = ''
+        """Convert a MessageChain into a list of component dicts.
+
+        Each dict has a ``type`` key (``'text'``, ``'image'``,
+        ``'voice'``, ``'file'``). Text items carry ``text``; media
+        items carry ``base64`` (may include a ``data:...;base64,``
+        prefix) and optionally ``name``.
+        """
+        items: list[dict] = []
        for msg in message_chain:
            if type(msg) is platform_message.Plain:
-                content += msg.text
-        return content
+                items.append({'type': 'text', 'text': msg.text})
+            elif type(msg) is platform_message.Image:
+                items.append({'type': 'image', 'base64': msg.base64 or ''})
+            elif type(msg) is platform_message.Voice:
+                items.append({'type': 'voice', 'base64': msg.base64 or ''})
+            elif type(msg) is platform_message.File:
+                items.append({'type': 'file', 'base64': msg.base64 or '', 'name': msg.name or ''})
+        return items

    @staticmethod
    async def target2yiri(event: WecomBotEvent, bot_name: str = ''):
@@ -362,13 +377,76 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
                }
            )

+    @staticmethod
+    def _join_text_components(items: list[dict]) -> str:
+        """Concatenate ``text`` items in order, leaving media items alone."""
+        return ''.join(item['text'] for item in items if item.get('type') == 'text')
+
+    @staticmethod
+    def _iter_media_components(items: list[dict]):
+        """Yield non-text items in order."""
+        for item in items:
+            if item.get('type') in {'image', 'voice', 'file'}:
+                yield item
+
+    @staticmethod
+    async def _send_media(
+        bot,
+        req_id: str,
+        item: dict,
+    ) -> bool:
+        """Upload *item* to the WeCom AI Bot CDN and send it as a media reply.
+
+        Returns True on success. Falls back to a no-op (with a warning log)
+        if the SDK does not yet implement ``upload_media`` /
+        ``reply_image`` / ``reply_file`` / ``reply_voice`` — the framework
+        will keep working, just without image delivery.
+        """
+        kind = item.get('type')
+        upload = getattr(bot, 'upload_media', None)
+        if upload is None:
+            return False
+        b64_text = item.get('base64') or ''
+        if not b64_text:
+            return False
+        if b64_text.startswith('data:') and ',' in b64_text:
+            b64_text = b64_text.split(',', 1)[1]
+        try:
+            data = base64.b64decode(b64_text, validate=False)
+        except Exception:
+            return False
+        if not data:
+            return False
+        try:
+            upload_result = await upload(data, item.get('name') or f'attachment.{kind}', media_type=kind)
+        except Exception:
+            return False
+        media_id = getattr(upload_result, 'media_id', None) or (
+            isinstance(upload_result, dict) and upload_result.get('media_id')
+        )
+        if not media_id:
+            return False
+        reply_fn = {
+            'image': getattr(bot, 'reply_image', None),
+            'file': getattr(bot, 'reply_file', None),
+            'voice': getattr(bot, 'reply_voice', None),
+        }.get(kind)
+        if reply_fn is None:
+            return False
+        try:
+            await reply_fn(req_id, media_id)
+            return True
+        except Exception:
+            return False
+
    async def reply_message(
        self,
        message_source: platform_events.MessageEvent,
        message: platform_message.MessageChain,
        quote_origin: bool = False,
    ):
-        content = await self.message_converter.yiri2target(message)
+        items = await self.message_converter.yiri2target(message)
+        text = self._join_text_components(items)
        _ws_mode = not self.config.get('enable-webhook', False)

        event = message_source.source_platform_object
@@ -382,7 +460,7 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
                else:
                    chat_id = str(message_source.sender.id)
                try:
-                    await self.bot.send_message(chat_id, content)
+                    await self.bot.send_message(chat_id, text)
                except Exception:
                    await self.logger.error(
                        f'WeComBot: proactive reply for synthetic event failed: {traceback.format_exc()}'
@@ -396,12 +474,15 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):

        if _ws_mode:
            req_id = event.get('req_id', '') if isinstance(event, dict) else getattr(event, 'req_id', '')
-            if req_id:
-                await self.bot.reply_text(req_id, content)
-            else:
-                await self.bot.set_message(event.message_id, content)
+            if text:
+                if req_id:
+                    await self.bot.reply_text(req_id, text)
+                else:
+                    await self.bot.set_message(event.message_id, text)
+            for item in self._iter_media_components(items):
+                await self._send_media(self.bot, req_id, item)
        else:
-            await self.bot.set_message(event.message_id, content)
+            await self.bot.set_message(event.message_id, text)

    async def reply_message_chunk(
        self,
@@ -411,7 +492,8 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
        quote_origin: bool = False,
        is_final: bool = False,
    ):
-        content = await self.message_converter.yiri2target(message)
+        items = await self.message_converter.yiri2target(message)
+        text = self._join_text_components(items)
        _ws_mode = not self.config.get('enable-webhook', False)

        # Synthetic events (e.g. button-click triggered form resume) have
@@ -420,7 +502,7 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
        # of the stream/reply path.
        spo = message_source.source_platform_object
        if spo is None:
-            return await self._handle_synthetic_chunk(message_source, bot_message, content, is_final, _ws_mode)
+            return await self._handle_synthetic_chunk(message_source, bot_message, text, is_final, _ws_mode)

        msg_id = spo.message_id

@@ -452,7 +534,7 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
                    form_data.get('actions', []) or [],
                )
            except Exception:
-                fallback = content or '（人工输入）'
+                fallback = text or '（人工输入）'
            if _ws_mode:
                event = message_source.source_platform_object
                req_id = event.get('req_id', '') if isinstance(event, dict) else getattr(event, 'req_id', '')
@@ -463,17 +545,22 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
            return {'stream': False, 'form': True, 'fallback': True}

        if _ws_mode:
-            success = await self.bot.push_stream_chunk(msg_id, content, is_final=is_final)
+            success = await self.bot.push_stream_chunk(msg_id, text, is_final=is_final)
            if not success and is_final:
                event = message_source.source_platform_object
                req_id = event.get('req_id', '')
                if req_id:
-                    await self.bot.reply_text(req_id, content)
+                    await self.bot.reply_text(req_id, text)
+            if is_final:
+                event = message_source.source_platform_object
+                req_id = event.get('req_id', '')
+                for item in self._iter_media_components(items):
+                    await self._send_media(self.bot, req_id, item)
            return {'stream': success}
        else:
-            success = await self.bot.push_stream_chunk(msg_id, content, is_final=is_final)
+            success = await self.bot.push_stream_chunk(msg_id, text, is_final=is_final)
            if not success and is_final:
-                await self.bot.set_message(msg_id, content)
+                await self.bot.set_message(msg_id, text)
            return {'stream': success}

    async def is_stream_output_supported(self) -> bool:
@@ -627,8 +714,9 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
    async def send_message(self, target_type, target_id, message):
        _ws_mode = not self.config.get('enable-webhook', False)
        if _ws_mode:
-            content = await self.message_converter.yiri2target(message)
-            await self.bot.send_message(target_id, content)
+            items = await self.message_converter.yiri2target(message)
+            text = self._join_text_components(items)
+            await self.bot.send_message(target_id, text)
        else:
            pass