fix(runner): avoid duplicate inline image payloads (#2631)

* fix(runner): avoid duplicate inline image payloads

* fix(runner): preserve attachments and complete HTTP agent delivery

Co-authored-by: multica-agent <[email protected]>

---------

Co-authored-by: Qingsong Li <[email protected]>
Co-authored-by: Chin Rock <[email protected]>
Co-authored-by: multica-agent <[email protected]>

src/langbot/pkg/box/runner.py

@@ -36,14 +36,24 @@ def prepare_input_files(query, agent_input):
     event envelope is metadata, never permission to read the Host filesystem.
     """
     from langbot_plugin.api.entities.builtin.runner.input import InputAttachment
-    from ..agent.runner.query_entry_adapter import QueryEntryAdapter
 
     originals = input_files(query)
+    # References are scoped to this query's previous preparation, not to the
+    # platform message order or to any caller-supplied filesystem path.
+    prepared = getattr(query, '_box_inputs', None)
+    previous_refs = {f'attachment-{i}': item for i, item in enumerate(prepared or [])}
     if not agent_input.attachments:
+        from ..agent.runner.query_entry_adapter import QueryEntryAdapter
+
         agent_input.attachments = [
             InputAttachment.model_validate(item)
             for item in QueryEntryAdapter._build_attachments(query, [c.model_dump() for c in agent_input.contents])
         ]
+    inline_images = {
+        content.image_base64
+        for content in (agent_input.contents or [])
+        if content.type == 'image_base64' and content.image_base64
+    }
     components = []
     for i, attachment in enumerate(agent_input.attachments):
         kind = (attachment.type or 'file').lower()
@@ -52,6 +62,11 @@ def prepare_input_files(query, agent_input):
         if cls is pm.File:
             fields.update(name=attachment.name or '', id=attachment.id or '')
         component = cls(**fields)
+        referenced = previous_refs.get(attachment.ref)
+        if type(referenced) is cls and not attachment.content and not attachment.url:
+            components.append(referenced)
+            attachment.ref, attachment.path = f'attachment-{i}', None
+            continue
         # Match by payload, never by list position (contents and message chains
         # can describe attachments in different orders).
         for original in originals:
@@ -77,6 +92,8 @@ def prepare_input_files(query, agent_input):
                 break
         components.append(component)
         attachment.ref, attachment.path = f'attachment-{i}', None
+        if kind == 'image' and attachment.content in inline_images:
+            attachment.content = None
     object.__setattr__(query, '_box_inputs', components)
 
 

src/langbot/pkg/platform/botmgr.py

@@ -34,6 +34,7 @@ from ..agent.runner.host_models import (
 )
 from ..agent.runner.resource_policy import ResourcePolicyProjector
 from ..agent.runner.platform_tools import resolve_agent_platform_tool_names
+from ..agent.runner.query_entry_adapter import QueryEntryAdapter
 from ..entity.persistence import workspace as persistence_workspace
 
 from ..api.http.context import ExecutionContext, PrincipalContext, PrincipalType, RequestContext

src/langbot/pkg/platform/sources/http_bot.py

@@ -89,7 +89,7 @@ class _SessionOutbound:
 
 
 class _SyncCollector:
-    """Collects reply parts for a /sync request and resolves when the turn ends."""
+    """Collects reply parts and signals receipt of a final message."""
 
     def __init__(self) -> None:
         self.parts: list = []
@@ -162,6 +162,9 @@ class HttpBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
     async def is_muted(self, group_id: int) -> bool:
         return False
 
+    def get_supported_apis(self) -> list[str]:
+        return ['send_message', 'reply_message']
+
     async def is_stream_output_supported(self) -> bool:
         return True
 
@@ -582,6 +585,12 @@ class HttpBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
             'message': [c.model_dump() if hasattr(c, 'model_dump') else c.__dict__ for c in message],
             'timestamp': datetime.now().isoformat(),
         }
+        collector = self.sync_waiters.get(str(target_id))
+        if collector is not None:
+            collector.parts.extend(payload['message'])
+            collector.done.set()
+            return payload
+
         await self._enqueue_callback(str(target_id), payload)
         return payload
 
@@ -612,6 +621,9 @@ class HttpBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
         Lossy by design (drops streaming/ordering nuance); documented as such.
         Concurrency-safe: routing is via the per-session ``_sync_waiters``
         registry that ``_emit_reply`` consults, not by patching methods.
+        A final message can precede Agent output-file delivery, so collection
+        also waits for the inbound listener to finish. Legacy queued pipelines
+        may finish their listener before sending any reply.
         """
         if session_id in self.sync_waiters:
             return self._err('duplicate', 'a sync request is already in flight for this session')
@@ -622,12 +634,19 @@ class HttpBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter):
         if listener_task is None:
             self.sync_waiters.pop(session_id, None)
             return self._err('overloaded', 'too many inbound messages are already being processed')
+
+        async def wait_for_turn():
+            await collector.done.wait()
+            await asyncio.shield(listener_task)
+
         try:
             timeout = int(self.config.get('callback_timeout', 15)) * 4
             try:
-                await asyncio.wait_for(collector.done.wait(), timeout=timeout)
+                await asyncio.wait_for(wait_for_turn(), timeout=timeout)
             except asyncio.TimeoutError:
                 await self.logger.warning(f'http_bot sync wait timed out for session {session_id}')
+            except Exception:
+                await self.logger.warning(f'http_bot sync listener failed for session {session_id}')
         finally:
             self.sync_waiters.pop(session_id, None)
             state = self.outbound_states.get(session_id)

src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py

@@ -24,7 +24,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.secret_managers.main import get_secret, get_secret_bool
 
 from .. import errors, reasoning, requester
-from ....utils import httpclient
+from ....utils import httpclient, image
 import langbot_plugin.api.entities.builtin.resource.tool as resource_tool
 import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query
 import langbot_plugin.api.entities.builtin.provider.message as provider_message
@@ -895,7 +895,9 @@ class LiteLLMRequester(requester.ProviderAPIRequester):
                             if isinstance(raw_image_url, dict):
                                 fallback_url = raw_image_url.get('url')
                         if image_b64 or fallback_url:
-                            part['image_url'] = {'url': image_b64 or fallback_url}
+                            part['image_url'] = {
+                                'url': image.normalize_image_data_url(image_b64) if image_b64 else fallback_url
+                            }
                             part['type'] = 'image_url'
                             part.pop('image_base64', None)
                         else:

src/langbot/pkg/utils/image.py

@@ -17,6 +17,30 @@ _INSECURE_SSL_CONTEXT.verify_mode = ssl.CERT_NONE
 DEFAULT_BASE64_MEDIA_LIMIT = 10 * 1024 * 1024
 
 
+def normalize_image_data_url(value: str) -> str:
+    """Wrap raw image base64 with its detected MIME type, never a guessed one.
+
+    Existing data URLs are preserved for provider-specific compatibility.
+    Bound decoding before inspecting untrusted image headers.
+    """
+    if value.startswith('data:'):
+        return value
+    if len(value) > 4 * ((DEFAULT_BASE64_MEDIA_LIMIT + 2) // 3):
+        raise ValueError('Base64 image exceeds the media size limit')
+    try:
+        decoded = base64.b64decode(value, validate=True)
+        if len(decoded) > DEFAULT_BASE64_MEDIA_LIMIT:
+            raise ValueError('Base64 image exceeds the media size limit')
+        with PIL.Image.open(io.BytesIO(decoded)) as image:
+            mime = PIL.Image.MIME.get(image.format or '')
+            image.verify()
+        if not mime or not mime.startswith('image/'):
+            raise ValueError('Unknown image MIME type')
+    except (ValueError, OSError, PIL.Image.DecompressionBombError) as exc:
+        raise ValueError('Invalid base64 image: expected encoded image bytes') from exc
+    return f'data:{mime};base64,{value}'
+
+
 def _detect_image_format(file_bytes: bytes) -> str:
     with PIL.Image.open(io.BytesIO(file_bytes)) as image:
         return str(image.format or 'jpeg').lower()