mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-22 10:17:13 +00:00
fix(wecombot): deliver sandbox outbox media through full pipeline chain (#2328)
* fix(wecombot): align media upload protocol * fix(wecombot): deliver outbox media in reply and fix tool call recording - Integrate _send_media into reply_message and reply_message_chunk so sandbox outbox images/voices/files are uploaded and sent instead of being silently dropped. - Add missing import base64 that caused _send_media to fail with a NameError swallowed by its except clause. - Change yiri2target to return component dicts (text/image/voice/file) so callers can distinguish text from media. - Fix _get_message_for_tool_context using result.first()/row[0] which returned a raw string instead of the ORM object, causing "'str' object has no attribute 'pipeline_id'" in tool call recording. Use result.scalars().first() per SQLAlchemy 2.0 convention. * fix(pipeline): collect outbox attachments on final chunk with empty content When the last streaming chunk has is_final=True but empty content (e.g. the LLM sends all text in earlier chunks), the 'if result.content' branch is skipped entirely, so _append_outbound_attachments never runs and sandbox outbox images are silently dropped. Add an elif branch for _is_final_assistant_message that creates an empty MessageChain and still collects outbox attachments, so images are delivered even when the final chunk carries no text. * fix(box): bypass stdout truncation when reading outbox via exec _read_outbox_via_exec used execute_tool which returns _serialize_result where stdout is truncated to output_limit_chars (4000). A 7KB JPEG encodes to ~9400 base64 chars, so the JSON payload was truncated and json.loads failed silently, returning an empty list. Call client.execute directly to get the raw BoxExecutionResult with untruncated stdout, so base64 file data is preserved. * fix(tests): adapt box and wrapper tests for client.execute and strict is_final check - wrapper.py: restrict outbox collection on empty-content chunks to actual MessageChunk instances with is_final=True, not generic Mock objects that happen to have role='assistant' - test_box_service.py: update _read_outbox_via_exec tests to mock client.execute (returning BoxExecutionResult) instead of execute_tool, matching the implementation change * chore(wecombot): remove temporary upload log * test(box): preserve direct outbox read and cleanup coverage --------- Co-authored-by: fdc310 <2213070223@qq.com> Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com>
This commit is contained in:
DongXiaoming
@@ -1210,8 +1210,9 @@ class BoxService:
|
||||
async def _read_outbox_via_exec(self, query: pipeline_query.Query) -> list[dict]:
|
||||
"""Fallback: read the outbox over the exec channel (E2B / remote).
|
||||
|
||||
Note: exec stdout is truncated by ``output_limit_chars``, so this path
|
||||
only reliably transfers small files. The host path is preferred.
|
||||
Uses ``client.execute`` directly (bypassing ``_serialize_result``)
|
||||
so stdout is NOT truncated by ``output_limit_chars`` - the raw
|
||||
base64 payload can be far larger than the 4000-char display limit.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
@@ -1265,14 +1266,22 @@ class BoxService:
|
||||
' break\n'
|
||||
'print(json.dumps(out))\n'
|
||||
)
|
||||
result = await self.execute_tool(
|
||||
{'command': f"python3 - <<'LBPY'\n{script}\nLBPY", 'timeout_sec': 120},
|
||||
query,
|
||||
)
|
||||
if not result.get('ok'):
|
||||
spec_payload: dict = {
|
||||
'cmd': f"python3 - <<'LBPY'\n{script}\nLBPY",
|
||||
'timeout_sec': 120,
|
||||
'session_id': self.resolve_box_session_id(query),
|
||||
}
|
||||
if 'extra_mounts' not in spec_payload:
|
||||
spec_payload['extra_mounts'] = self.build_skill_extra_mounts(query)
|
||||
try:
|
||||
spec = self.build_spec(spec_payload)
|
||||
result = await self.client.execute(spec)
|
||||
except Exception:
|
||||
return []
|
||||
if not result.ok:
|
||||
return []
|
||||
try:
|
||||
return _json.loads(str(result.get('stdout') or '').strip().splitlines()[-1])
|
||||
return _json.loads(str(result.stdout or '').strip().splitlines()[-1])
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
@@ -158,6 +158,18 @@ class ResponseWrapper(stage.PipelineStage):
|
||||
result_type=entities.ResultType.CONTINUE,
|
||||
new_query=query,
|
||||
)
|
||||
elif (
|
||||
isinstance(result, provider_message.MessageChunk) and result.is_final and not result.tool_calls
|
||||
):
|
||||
# Final streaming chunk with no text content but
|
||||
# possibly carrying sandbox outbox attachments.
|
||||
reply_chain = platform_message.MessageChain([])
|
||||
await self._append_outbound_attachments(query, reply_chain)
|
||||
query.resp_message_chain.append(reply_chain)
|
||||
yield entities.StageProcessResult(
|
||||
result_type=entities.ResultType.CONTINUE,
|
||||
new_query=query,
|
||||
)
|
||||
|
||||
if result.tool_calls is not None and len(result.tool_calls) > 0: # 有函数调用
|
||||
function_names = [tc.function.name for tc in result.tool_calls]
|
||||
|
||||
@@ -3,8 +3,10 @@ import typing
|
||||
import asyncio
|
||||
import time
|
||||
import traceback
|
||||
import base64
|
||||
|
||||
import datetime
|
||||
|
||||
import langbot_plugin.api.definition.abstract.platform.adapter as abstract_platform_adapter
|
||||
import langbot_plugin.api.entities.builtin.platform.message as platform_message
|
||||
import langbot_plugin.api.entities.builtin.platform.events as platform_events
|
||||
@@ -24,11 +26,24 @@ from langbot.libs.wecom_ai_bot_api.ws_client import WecomBotWsClient
|
||||
class WecomBotMessageConverter(abstract_platform_adapter.AbstractMessageConverter):
|
||||
@staticmethod
|
||||
async def yiri2target(message_chain: platform_message.MessageChain):
|
||||
content = ''
|
||||
"""Convert a MessageChain into a list of component dicts.
|
||||
|
||||
Each dict has a ``type`` key (``'text'``, ``'image'``,
|
||||
``'voice'``, ``'file'``). Text items carry ``text``; media
|
||||
items carry ``base64`` (may include a ``data:...;base64,``
|
||||
prefix) and optionally ``name``.
|
||||
"""
|
||||
items: list[dict] = []
|
||||
for msg in message_chain:
|
||||
if type(msg) is platform_message.Plain:
|
||||
content += msg.text
|
||||
return content
|
||||
items.append({'type': 'text', 'text': msg.text})
|
||||
elif type(msg) is platform_message.Image:
|
||||
items.append({'type': 'image', 'base64': msg.base64 or ''})
|
||||
elif type(msg) is platform_message.Voice:
|
||||
items.append({'type': 'voice', 'base64': msg.base64 or ''})
|
||||
elif type(msg) is platform_message.File:
|
||||
items.append({'type': 'file', 'base64': msg.base64 or '', 'name': msg.name or ''})
|
||||
return items
|
||||
|
||||
@staticmethod
|
||||
async def target2yiri(event: WecomBotEvent, bot_name: str = ''):
|
||||
@@ -362,13 +377,76 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _join_text_components(items: list[dict]) -> str:
|
||||
"""Concatenate ``text`` items in order, leaving media items alone."""
|
||||
return ''.join(item['text'] for item in items if item.get('type') == 'text')
|
||||
|
||||
@staticmethod
|
||||
def _iter_media_components(items: list[dict]):
|
||||
"""Yield non-text items in order."""
|
||||
for item in items:
|
||||
if item.get('type') in {'image', 'voice', 'file'}:
|
||||
yield item
|
||||
|
||||
@staticmethod
|
||||
async def _send_media(
|
||||
bot,
|
||||
req_id: str,
|
||||
item: dict,
|
||||
) -> bool:
|
||||
"""Upload *item* to the WeCom AI Bot CDN and send it as a media reply.
|
||||
|
||||
Returns True on success. Falls back to a no-op (with a warning log)
|
||||
if the SDK does not yet implement ``upload_media`` /
|
||||
``reply_image`` / ``reply_file`` / ``reply_voice`` — the framework
|
||||
will keep working, just without image delivery.
|
||||
"""
|
||||
kind = item.get('type')
|
||||
upload = getattr(bot, 'upload_media', None)
|
||||
if upload is None:
|
||||
return False
|
||||
b64_text = item.get('base64') or ''
|
||||
if not b64_text:
|
||||
return False
|
||||
if b64_text.startswith('data:') and ',' in b64_text:
|
||||
b64_text = b64_text.split(',', 1)[1]
|
||||
try:
|
||||
data = base64.b64decode(b64_text, validate=False)
|
||||
except Exception:
|
||||
return False
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
upload_result = await upload(data, item.get('name') or f'attachment.{kind}', media_type=kind)
|
||||
except Exception:
|
||||
return False
|
||||
media_id = getattr(upload_result, 'media_id', None) or (
|
||||
isinstance(upload_result, dict) and upload_result.get('media_id')
|
||||
)
|
||||
if not media_id:
|
||||
return False
|
||||
reply_fn = {
|
||||
'image': getattr(bot, 'reply_image', None),
|
||||
'file': getattr(bot, 'reply_file', None),
|
||||
'voice': getattr(bot, 'reply_voice', None),
|
||||
}.get(kind)
|
||||
if reply_fn is None:
|
||||
return False
|
||||
try:
|
||||
await reply_fn(req_id, media_id)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def reply_message(
|
||||
self,
|
||||
message_source: platform_events.MessageEvent,
|
||||
message: platform_message.MessageChain,
|
||||
quote_origin: bool = False,
|
||||
):
|
||||
content = await self.message_converter.yiri2target(message)
|
||||
items = await self.message_converter.yiri2target(message)
|
||||
text = self._join_text_components(items)
|
||||
_ws_mode = not self.config.get('enable-webhook', False)
|
||||
|
||||
event = message_source.source_platform_object
|
||||
@@ -382,7 +460,7 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
|
||||
else:
|
||||
chat_id = str(message_source.sender.id)
|
||||
try:
|
||||
await self.bot.send_message(chat_id, content)
|
||||
await self.bot.send_message(chat_id, text)
|
||||
except Exception:
|
||||
await self.logger.error(
|
||||
f'WeComBot: proactive reply for synthetic event failed: {traceback.format_exc()}'
|
||||
@@ -396,12 +474,15 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
|
||||
|
||||
if _ws_mode:
|
||||
req_id = event.get('req_id', '') if isinstance(event, dict) else getattr(event, 'req_id', '')
|
||||
if req_id:
|
||||
await self.bot.reply_text(req_id, content)
|
||||
else:
|
||||
await self.bot.set_message(event.message_id, content)
|
||||
if text:
|
||||
if req_id:
|
||||
await self.bot.reply_text(req_id, text)
|
||||
else:
|
||||
await self.bot.set_message(event.message_id, text)
|
||||
for item in self._iter_media_components(items):
|
||||
await self._send_media(self.bot, req_id, item)
|
||||
else:
|
||||
await self.bot.set_message(event.message_id, content)
|
||||
await self.bot.set_message(event.message_id, text)
|
||||
|
||||
async def reply_message_chunk(
|
||||
self,
|
||||
@@ -411,7 +492,8 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
|
||||
quote_origin: bool = False,
|
||||
is_final: bool = False,
|
||||
):
|
||||
content = await self.message_converter.yiri2target(message)
|
||||
items = await self.message_converter.yiri2target(message)
|
||||
text = self._join_text_components(items)
|
||||
_ws_mode = not self.config.get('enable-webhook', False)
|
||||
|
||||
# Synthetic events (e.g. button-click triggered form resume) have
|
||||
@@ -420,7 +502,7 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
|
||||
# of the stream/reply path.
|
||||
spo = message_source.source_platform_object
|
||||
if spo is None:
|
||||
return await self._handle_synthetic_chunk(message_source, bot_message, content, is_final, _ws_mode)
|
||||
return await self._handle_synthetic_chunk(message_source, bot_message, text, is_final, _ws_mode)
|
||||
|
||||
msg_id = spo.message_id
|
||||
|
||||
@@ -452,7 +534,7 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
|
||||
form_data.get('actions', []) or [],
|
||||
)
|
||||
except Exception:
|
||||
fallback = content or '（人工输入）'
|
||||
fallback = text or '（人工输入）'
|
||||
if _ws_mode:
|
||||
event = message_source.source_platform_object
|
||||
req_id = event.get('req_id', '') if isinstance(event, dict) else getattr(event, 'req_id', '')
|
||||
@@ -463,17 +545,22 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
|
||||
return {'stream': False, 'form': True, 'fallback': True}
|
||||
|
||||
if _ws_mode:
|
||||
success = await self.bot.push_stream_chunk(msg_id, content, is_final=is_final)
|
||||
success = await self.bot.push_stream_chunk(msg_id, text, is_final=is_final)
|
||||
if not success and is_final:
|
||||
event = message_source.source_platform_object
|
||||
req_id = event.get('req_id', '')
|
||||
if req_id:
|
||||
await self.bot.reply_text(req_id, content)
|
||||
await self.bot.reply_text(req_id, text)
|
||||
if is_final:
|
||||
event = message_source.source_platform_object
|
||||
req_id = event.get('req_id', '')
|
||||
for item in self._iter_media_components(items):
|
||||
await self._send_media(self.bot, req_id, item)
|
||||
return {'stream': success}
|
||||
else:
|
||||
success = await self.bot.push_stream_chunk(msg_id, content, is_final=is_final)
|
||||
success = await self.bot.push_stream_chunk(msg_id, text, is_final=is_final)
|
||||
if not success and is_final:
|
||||
await self.bot.set_message(msg_id, content)
|
||||
await self.bot.set_message(msg_id, text)
|
||||
return {'stream': success}
|
||||
|
||||
async def is_stream_output_supported(self) -> bool:
|
||||
@@ -627,8 +714,9 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
|
||||
async def send_message(self, target_type, target_id, message):
|
||||
_ws_mode = not self.config.get('enable-webhook', False)
|
||||
if _ws_mode:
|
||||
content = await self.message_converter.yiri2target(message)
|
||||
await self.bot.send_message(target_id, content)
|
||||
items = await self.message_converter.yiri2target(message)
|
||||
text = self._join_text_components(items)
|
||||
await self.bot.send_message(target_id, text)
|
||||
else:
|
||||
pass
|
||||
|
||||
|
||||
Reference in New Issue
Block a user