mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-06-18 19:44:21 +00:00
Junyan Chin
a1e6eccdeb
* feat(box): bidirectional attachment transfer for sandbox Materialize inbound attachments into the sandbox workspace so agents can process user-sent files, and collect agent-produced files from the outbox to attach them back to the reply. - box(service): add materialize_inbound_attachments / collect_outbound attachments. Prefer direct host-filesystem read/write on the bind-mounted workspace (no size limit), falling back to chunked exec only for non-shared backends (e2b/remote). Clear per-query inbox/outbox dirs at turn start to avoid query_id-reuse collisions. - provider(localagent): inject inbound attachment descriptors into the sandbox and append a system note telling the agent the inbox/outbox paths. - pipeline(wrapper): collect outbox files on the final stream chunk and append them as attachment components to the response chain. - web(debug-dialog): render File components with a download link when base64/url is present; add base64/path fields to the File entity. - tests: cover inbound/outbound, large-file transfer without truncation, and stale-dir clearing (86 passing). * feat(box): support voice/file attachment round-trip end-to-end Extends the bidirectional attachment transfer to audio and arbitrary files through the real webchat UI, and fixes the model-payload errors that non-image attachments triggered. - platform(websocket_adapter): resolve Voice/File component storage keys to base64 (previously only Image), so audio/documents reach the sandbox inbox. - web(debug-dialog): accept audio/* and any file in the uploader (was image-only), classify by mimetype, upload Voice/File via the documents endpoint, and render non-image staged attachments as a chip. - provider(litellmchat): drop non-image file parts (file_base64 / file_url) when building the OpenAI/LiteLLM payload. These come from Voice/File attachments — including ones replayed from conversation history — and the agent reads their bytes from the sandbox, not the model. Without this the provider rejects the request: 'invalid content type=file_base64'. - provider(localagent): also strip those parts from the current user message alongside the sandbox-path note (model-facing clarity; the requester is the real safety net for history). - tests: cover the requester strip/keep behavior (file dropped, image kept and reshaped to image_url, mixed history, plain-string content). * test(box): cover inbound/outbound attachment helpers; fix ruff format - ruff format localagent.py (CI ruff format --check was failing) - add unit tests for ResponseWrapper outbound-attachment helpers (wrapper.py 78%->98%) - add unit tests for LocalAgentRunner._inject_inbound_attachments - add unit tests for WebSocketAdapter._process_image_components (0%->covered) Lifts PR patch coverage from 68.97% to ~88% (>75% target).
93 lines
3.3 KiB
Python
93 lines
3.3 KiB
Python
"""Unit tests for WebSocketAdapter._process_image_components.
|
|
|
|
The web debug client uploads Image / Voice / File components carrying a storage
|
|
key in ``path``. This helper resolves each to a base64 data URI (so multimodal
|
|
LLM input and the Box sandbox inbox have usable bytes), then deletes the
|
|
consumed storage object and clears ``path``. Covers mimetype selection per
|
|
type and graceful error handling.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
from unittest.mock import AsyncMock, Mock
|
|
|
|
import pytest
|
|
|
|
from langbot.pkg.platform.sources.websocket_adapter import WebSocketAdapter
|
|
|
|
|
|
def _make_adapter(load_return=b'hello', load_side_effect=None):
|
|
provider = Mock()
|
|
provider.load = AsyncMock(return_value=load_return, side_effect=load_side_effect)
|
|
provider.delete = AsyncMock()
|
|
ap = Mock()
|
|
ap.storage_mgr.storage_provider = provider
|
|
logger = Mock()
|
|
logger.error = AsyncMock()
|
|
# WebSocketAdapter is a pydantic model; bypass full __init__/validation.
|
|
adapter = WebSocketAdapter.model_construct(ap=ap, logger=logger)
|
|
return adapter, provider
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_image_jpeg_mimetype_and_cleanup():
|
|
adapter, provider = _make_adapter(load_return=b'\xff\xd8\xff')
|
|
chain = [{'type': 'Image', 'path': 'storage://abc/photo.jpg'}]
|
|
|
|
await adapter._process_image_components(chain)
|
|
|
|
expected_b64 = base64.b64encode(b'\xff\xd8\xff').decode('utf-8')
|
|
assert chain[0]['base64'] == f'data:image/jpeg;base64,{expected_b64}'
|
|
assert chain[0]['path'] == '' # consumed
|
|
provider.delete.assert_awaited_once_with('storage://abc/photo.jpg')
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_image_defaults_to_png():
|
|
adapter, _ = _make_adapter()
|
|
chain = [{'type': 'Image', 'path': 'storage://abc/blob'}]
|
|
await adapter._process_image_components(chain)
|
|
assert chain[0]['base64'].startswith('data:image/png;base64,')
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_voice_uses_guessed_or_wav_mimetype():
|
|
adapter, _ = _make_adapter()
|
|
chain = [{'type': 'Voice', 'path': 'storage://abc/clip.wav'}]
|
|
await adapter._process_image_components(chain)
|
|
assert chain[0]['base64'].startswith('data:audio/')
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_file_uses_octet_stream_fallback():
|
|
adapter, _ = _make_adapter()
|
|
chain = [{'type': 'File', 'path': 'storage://abc/unknownblob'}]
|
|
await adapter._process_image_components(chain)
|
|
assert chain[0]['base64'].startswith('data:application/octet-stream;base64,')
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skips_components_without_path_or_unknown_type():
|
|
adapter, provider = _make_adapter()
|
|
chain = [
|
|
{'type': 'Image', 'path': ''}, # no path
|
|
{'type': 'Plain', 'path': 'storage://abc/x'}, # not a file component
|
|
{'type': 'At', 'target': '123'}, # no path key at all
|
|
]
|
|
await adapter._process_image_components(chain)
|
|
provider.load.assert_not_awaited()
|
|
assert 'base64' not in chain[0]
|
|
assert 'base64' not in chain[1]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_load_failure_is_logged_not_raised():
|
|
adapter, _ = _make_adapter(load_side_effect=RuntimeError('storage down'))
|
|
chain = [{'type': 'File', 'path': 'storage://abc/doc.pdf'}]
|
|
|
|
# must not raise
|
|
await adapter._process_image_components(chain)
|
|
assert 'base64' not in chain[0]
|
|
adapter.logger.error.assert_awaited_once()
|