mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-09 12:40:59 +00:00
feat(box): bidirectional attachment transfer for sandbox (#2257)
* feat(box): bidirectional attachment transfer for sandbox Materialize inbound attachments into the sandbox workspace so agents can process user-sent files, and collect agent-produced files from the outbox to attach them back to the reply. - box(service): add materialize_inbound_attachments / collect_outbound attachments. Prefer direct host-filesystem read/write on the bind-mounted workspace (no size limit), falling back to chunked exec only for non-shared backends (e2b/remote). Clear per-query inbox/outbox dirs at turn start to avoid query_id-reuse collisions. - provider(localagent): inject inbound attachment descriptors into the sandbox and append a system note telling the agent the inbox/outbox paths. - pipeline(wrapper): collect outbox files on the final stream chunk and append them as attachment components to the response chain. - web(debug-dialog): render File components with a download link when base64/url is present; add base64/path fields to the File entity. - tests: cover inbound/outbound, large-file transfer without truncation, and stale-dir clearing (86 passing). * feat(box): support voice/file attachment round-trip end-to-end Extends the bidirectional attachment transfer to audio and arbitrary files through the real webchat UI, and fixes the model-payload errors that non-image attachments triggered. - platform(websocket_adapter): resolve Voice/File component storage keys to base64 (previously only Image), so audio/documents reach the sandbox inbox. - web(debug-dialog): accept audio/* and any file in the uploader (was image-only), classify by mimetype, upload Voice/File via the documents endpoint, and render non-image staged attachments as a chip. - provider(litellmchat): drop non-image file parts (file_base64 / file_url) when building the OpenAI/LiteLLM payload. These come from Voice/File attachments — including ones replayed from conversation history — and the agent reads their bytes from the sandbox, not the model. Without this the provider rejects the request: 'invalid content type=file_base64'. - provider(localagent): also strip those parts from the current user message alongside the sandbox-path note (model-facing clarity; the requester is the real safety net for history). - tests: cover the requester strip/keep behavior (file dropped, image kept and reshaped to image_url, mixed history, plain-string content). * test(box): cover inbound/outbound attachment helpers; fix ruff format - ruff format localagent.py (CI ruff format --check was failing) - add unit tests for ResponseWrapper outbound-attachment helpers (wrapper.py 78%->98%) - add unit tests for LocalAgentRunner._inject_inbound_attachments - add unit tests for WebSocketAdapter._process_image_components (0%->covered) Lifts PR patch coverage from 68.97% to ~88% (>75% target).
This commit is contained in:
@@ -0,0 +1,146 @@
|
||||
"""Unit tests for ResponseWrapper outbound-attachment helpers.
|
||||
|
||||
Covers the sandbox -> user attachment path added for the Box attachment
|
||||
round-trip:
|
||||
|
||||
* ``_is_final_assistant_message`` — only the terminal, tool-call-free assistant
|
||||
message (or a final MessageChunk) should trigger collection.
|
||||
* ``_append_outbound_attachments`` — collects sandbox outbox files exactly once
|
||||
per query and maps each descriptor to the right platform component, swallowing
|
||||
collection errors.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
import langbot_plugin.api.entities.builtin.platform.message as platform_message
|
||||
import langbot_plugin.api.entities.builtin.provider.message as provider_message
|
||||
|
||||
from langbot.pkg.pipeline.wrapper.wrapper import ResponseWrapper
|
||||
|
||||
|
||||
def _make_wrapper(box_service) -> ResponseWrapper:
|
||||
app = SimpleNamespace(logger=Mock())
|
||||
wrapper = ResponseWrapper.__new__(ResponseWrapper)
|
||||
wrapper.ap = app
|
||||
return wrapper
|
||||
|
||||
|
||||
def _make_query():
|
||||
return SimpleNamespace(variables={})
|
||||
|
||||
|
||||
def test_is_final_assistant_message_plain_assistant():
|
||||
wrapper = _make_wrapper(box_service=None)
|
||||
msg = provider_message.Message(role='assistant', content='done')
|
||||
assert wrapper._is_final_assistant_message(msg) is True
|
||||
|
||||
|
||||
def test_is_final_assistant_message_rejects_non_assistant():
|
||||
wrapper = _make_wrapper(box_service=None)
|
||||
msg = provider_message.Message(role='tool', content='{}')
|
||||
assert wrapper._is_final_assistant_message(msg) is False
|
||||
|
||||
|
||||
def test_is_final_assistant_message_rejects_tool_call_round():
|
||||
wrapper = _make_wrapper(box_service=None)
|
||||
msg = provider_message.Message(
|
||||
role='assistant',
|
||||
content='calling',
|
||||
tool_calls=[
|
||||
provider_message.ToolCall(
|
||||
id='c1',
|
||||
type='function',
|
||||
function=provider_message.FunctionCall(name='exec', arguments='{}'),
|
||||
)
|
||||
],
|
||||
)
|
||||
assert wrapper._is_final_assistant_message(msg) is False
|
||||
|
||||
|
||||
def test_is_final_assistant_message_non_final_chunk():
|
||||
wrapper = _make_wrapper(box_service=None)
|
||||
chunk = provider_message.MessageChunk(role='assistant', content='partial', is_final=False)
|
||||
assert wrapper._is_final_assistant_message(chunk) is False
|
||||
|
||||
final_chunk = provider_message.MessageChunk(role='assistant', content='partial', is_final=True)
|
||||
assert wrapper._is_final_assistant_message(final_chunk) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_outbound_attachments_maps_each_type():
|
||||
box_service = SimpleNamespace(
|
||||
available=True,
|
||||
collect_outbound_attachments=AsyncMock(
|
||||
return_value=[
|
||||
{'type': 'Image', 'base64': 'data:image/png;base64,iVBORw0K'},
|
||||
{'type': 'Voice', 'base64': 'data:audio/wav;base64,UklGRg=='},
|
||||
{'type': 'File', 'name': 'report.xlsx', 'base64': 'data:app;base64,UEsDBA=='},
|
||||
]
|
||||
),
|
||||
)
|
||||
wrapper = _make_wrapper(box_service)
|
||||
wrapper.ap.box_service = box_service
|
||||
query = _make_query()
|
||||
chain = platform_message.MessageChain([])
|
||||
|
||||
await wrapper._append_outbound_attachments(query, chain)
|
||||
|
||||
kinds = [type(c).__name__ for c in chain]
|
||||
assert kinds == ['Image', 'Voice', 'File']
|
||||
assert query.variables['_sandbox_outbound_collected'] is True
|
||||
# File keeps its name
|
||||
file_comp = chain[2]
|
||||
assert getattr(file_comp, 'name', None) == 'report.xlsx'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_outbound_attachments_runs_once_per_query():
|
||||
box_service = SimpleNamespace(
|
||||
available=True,
|
||||
collect_outbound_attachments=AsyncMock(return_value=[]),
|
||||
)
|
||||
wrapper = _make_wrapper(box_service)
|
||||
wrapper.ap.box_service = box_service
|
||||
query = _make_query()
|
||||
query.variables['_sandbox_outbound_collected'] = True
|
||||
chain = platform_message.MessageChain([])
|
||||
|
||||
await wrapper._append_outbound_attachments(query, chain)
|
||||
|
||||
box_service.collect_outbound_attachments.assert_not_awaited()
|
||||
assert len(chain) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_outbound_attachments_noop_without_box_service():
|
||||
wrapper = _make_wrapper(box_service=None)
|
||||
wrapper.ap.box_service = None
|
||||
query = _make_query()
|
||||
chain = platform_message.MessageChain([])
|
||||
|
||||
await wrapper._append_outbound_attachments(query, chain)
|
||||
assert len(chain) == 0
|
||||
# not marked collected, since service is unavailable
|
||||
assert '_sandbox_outbound_collected' not in query.variables
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_outbound_attachments_swallows_collection_error():
|
||||
box_service = SimpleNamespace(
|
||||
available=True,
|
||||
collect_outbound_attachments=AsyncMock(side_effect=RuntimeError('boom')),
|
||||
)
|
||||
wrapper = _make_wrapper(box_service)
|
||||
wrapper.ap.box_service = box_service
|
||||
query = _make_query()
|
||||
chain = platform_message.MessageChain([])
|
||||
|
||||
# must not raise
|
||||
await wrapper._append_outbound_attachments(query, chain)
|
||||
assert len(chain) == 0
|
||||
wrapper.ap.logger.warning.assert_called_once()
|
||||
Reference in New Issue
Block a user