Merge pull request #2693 from langbot-app/test/quality-box-platform-oracles-20261017

test(runtime): verify sandbox and platform behavior contracts
2026-10-23 17:05:28 +08:00 · 2026-10-17 20:12:22 +08:00
parent c9f886a203 c03f33d855
commit fc877f73df
9 changed files with 305 additions and 103 deletions
@@ -434,7 +434,7 @@ async def test_box_service_get_sessions_delegates_to_client():
    sessions = await service.get_sessions(_CONTEXT)

    assert sessions == [{'session_id': 'test-session'}]
-    client.get_sessions.assert_awaited_once()
+    client.get_sessions.assert_awaited_once_with(action_context=_ACTION_CONTEXT)


@pytest.mark.asyncio
@@ -599,12 +599,14 @@ async def test_cloud_box_service_reconnect_restores_runtime_only(
    connector.reconnect = AsyncMock()
    service._ensure_default_workspace = Mock()
    service._verify_cloud_runtime = AsyncMock()
+    service._purge_attachment_dirs = AsyncMock()
    monkeypatch.setattr('langbot.pkg.box.service.asyncio.sleep', AsyncMock())

    await service._reconnect_loop(connector)

    connector.reconnect.assert_awaited_once()
    service._verify_cloud_runtime.assert_awaited_once()
+    service._purge_attachment_dirs.assert_not_awaited()
    assert service.available is True


@@ -674,6 +676,8 @@ async def test_box_service_allows_host_mount_under_configured_root(tmp_path):

    assert result['ok'] is True
    assert backend.start_calls == ['person_test_user']
+    assert backend.start_specs[0].host_path == os.path.realpath(host_dir)
+    assert backend.start_specs[0].host_path_mode == BoxHostMountMode.READ_WRITE


@pytest.mark.asyncio
@@ -744,16 +748,20 @@ async def test_box_service_rejects_host_mount_outside_allowed_roots(tmp_path):
    service = BoxService(make_app(logger, [str(allowed_root)]), client=_InProcessBoxRuntimeClient(logger, runtime))
    await service.initialize()

-    with pytest.raises(BoxValidationError):
+    with pytest.raises(BoxValidationError, match='outside allowed_mount_roots'):
        await service.execute_spec_payload(
            {
                'cmd': 'pwd',
                'host_path': str(disallowed_root),
-                'session_id': '12',
+                'session_id': 'person_test_user',
            },
            make_query(12),
        )

+    assert backend.start_calls == []
+    assert backend.exec_calls == []
+    assert list(disallowed_root.iterdir()) == []
+

@pytest.mark.asyncio
 async def test_box_runtime_rejects_host_mount_conflict_in_same_session(tmp_path):
@@ -1064,8 +1072,18 @@ def test_builtin_profiles_are_consistent():


@pytest.mark.asyncio
-async def test_profile_default_applies_resource_limits():
+async def test_profile_default_applies_resource_limits(monkeypatch):
    """Default profile resource limits are applied to BoxSpec."""
+    profile = BUILTIN_PROFILES['default'].model_copy(
+        update={
+            'cpus': 0.75,
+            'memory_mb': 320,
+            'pids_limit': 37,
+            'read_only_rootfs': False,
+            'workspace_quota_mb': 19,
+        }
+    )
+    monkeypatch.setitem(BUILTIN_PROFILES, 'default', profile)
    logger = Mock()
    backend = FakeBackend(logger)
    runtime = BoxRuntime(logger=logger, backends=[backend], session_ttl_sec=300)
@@ -1473,8 +1491,14 @@ async def test_rpc_client_get_status():
    try:
        status = await client.get_status(action_context=_ACTION_CONTEXT)

-        assert 'backend' in status
-        assert 'active_sessions' in status
+        assert status['backend']['name'] == 'fake'
+        assert status['backend']['available'] is True
+        assert status['active_sessions'] == 0
+        await client.execute(BoxSpec(cmd='true', session_id='status-session'), action_context=_ACTION_CONTEXT)
+        status = await client.get_status(action_context=_ACTION_CONTEXT)
+        assert status['active_sessions'] == 1
+        assert status['managed_processes'] == 0
+        assert status['session_ttl_sec'] == 300
    finally:
        server_task.cancel()
        client_task.cancel()
@@ -1515,8 +1539,10 @@ async def test_rpc_client_delete_session():
        spec = BoxSpec.model_validate({'cmd': 'echo hi', 'session_id': 'r-del-1'})
        await client.execute(spec, action_context=_ACTION_CONTEXT)

+        physical_session_id = backend.start_calls[0]
        await client.delete_session('r-del-1', action_context=_ACTION_CONTEXT)

+        assert backend.stop_calls == [physical_session_id]
        sessions = await client.get_sessions(action_context=_ACTION_CONTEXT)
        assert len(sessions) == 0
    finally:
@@ -1958,13 +1984,26 @@ class TestAttachmentHelpers:


 class TestInboundOutboundRoundTrip:
+    @staticmethod
+    async def _execute_command(command, workspace):
+        process = await asyncio.create_subprocess_exec(
+            '/bin/sh',
+            '-c',
+            command.replace('/workspace', str(workspace)),
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, stderr = await process.communicate()
+        assert process.returncode == 0, stderr.decode()
+        return stdout.decode().replace(str(workspace), '/workspace')
+
    def _service(self) -> BoxService:
        service = BoxService(make_app(Mock()), client=Mock(spec=BoxRuntimeClient))
        service._available = True
        return service

    @pytest.mark.asyncio
-    async def test_materialize_inbound_writes_and_describes(self):
+    async def test_materialize_inbound_writes_and_describes(self, tmp_path):
        import base64

        import langbot_plugin.api.entities.builtin.platform.message as platform_message
@@ -1982,14 +2021,9 @@ class TestInboundOutboundRoundTrip:
            ]
        )

-        # Mock the sandbox write path: echo back the written paths.
        async def fake_execute_tool(parameters, q):
-            assert '/workspace/inbox/' in parameters['command']
-            return {
-                'ok': True,
-                'stdout': '["/workspace/inbox/query-42/image_1.png"]',
-                'stderr': '',
-            }
+            assert q is query
+            return {'ok': True, 'stdout': await self._execute_command(parameters['command'], tmp_path), 'stderr': ''}

        service.execute_tool = AsyncMock(side_effect=fake_execute_tool)

@@ -1999,6 +2033,7 @@ class TestInboundOutboundRoundTrip:
        assert d['type'] == 'Image'
        assert d['path'] == '/workspace/inbox/query-42/image_1.png'
        assert d['size'] == len(img_bytes)
+        assert (tmp_path / 'inbox/query-42/image_1.png').read_bytes() == img_bytes

    @pytest.mark.asyncio
    async def test_materialize_inbound_noop_without_attachments(self):
@@ -2012,45 +2047,40 @@ class TestInboundOutboundRoundTrip:
        service.execute_tool.assert_not_called()

    @pytest.mark.asyncio
-    async def test_collect_outbound_reads_and_clears(self):
+    async def test_collect_outbound_reads_and_clears(self, tmp_path):
        service = self._service()
        query = make_query()
-
-        calls = []
+        outbox = tmp_path / 'outbox/query-42'
+        outbox.mkdir(parents=True)
+        (outbox / 'out.png').write_bytes(b'ABC')
+        guard = tmp_path / 'outbox/query-other/protected.txt'
+        guard.parent.mkdir()
+        guard.write_bytes(b'other query')

        async def fake_client_execute(spec, *, action_context=None):
-            assert action_context.workspace_uuid == query.workspace_uuid
-            cmd = spec.cmd
-            calls.append(cmd)
-            if 'os.scandir' in cmd:
-                return BoxExecutionResult(
-                    session_id='s',
-                    backend_name='test',
-                    status=BoxExecutionStatus.COMPLETED,
-                    exit_code=0,
-                    stdout='[{"name": "out.png", "b64": "QUJD"}]',
-                    duration_ms=10,
-                )
-            # the rm -rf cleanup call
+            assert action_context == _ACTION_CONTEXT
            return BoxExecutionResult(
                session_id='s',
                backend_name='test',
                status=BoxExecutionStatus.COMPLETED,
                exit_code=0,
-                stdout='',
+                stdout=await self._execute_command(spec.cmd, tmp_path),
                duration_ms=10,
            )

+        async def fake_execute_tool(parameters, q):
+            assert q is query
+            return {'ok': True, 'stdout': await self._execute_command(parameters['command'], tmp_path), 'stderr': ''}
+
        service.client.execute = AsyncMock(side_effect=fake_client_execute)
-        service.execute_tool = AsyncMock(return_value={'ok': True, 'stdout': '', 'stderr': ''})
+        service.execute_tool = AsyncMock(side_effect=fake_execute_tool)

        attachments = await service.collect_outbound_attachments(query)
-        assert len(attachments) == 1
-        assert attachments[0]['type'] == 'Image'
-        assert attachments[0]['name'] == 'out.png'
-        # cleanup (rm -rf) must have been issued after a successful collection
-        service.execute_tool.assert_awaited_once()
-        assert 'rm -rf' in service.execute_tool.await_args.args[0]['command']
+
+        assert attachments == [{'type': 'Image', 'name': 'out.png', 'base64': 'data:image/png;base64,QUJD'}]
+        assert not (outbox / 'out.png').exists()
+        assert guard.read_bytes() == b'other query'
+        assert await service.collect_outbound_attachments(query) == []

    @pytest.mark.asyncio
    async def test_collect_outbound_empty_still_clears(self):
@@ -2157,6 +2187,7 @@ class TestAttachmentHostPath:
        stale_dir = os.path.join(ws, 'inbox', 'query-42')
        os.makedirs(stale_dir, exist_ok=True)
        pathlib.Path(stale_dir, 'image_1.png').write_bytes(b'STALE-OLD-IMAGE')
+        pathlib.Path(stale_dir, 'leftover-secret.txt').write_bytes(b'STALE-OTHER-FILE')

        new = b'\x89PNG\r\n\x1a\n NEW'
        b64 = 'data:image/png;base64,' + base64.b64encode(new).decode()
@@ -2170,6 +2201,7 @@ class TestAttachmentHostPath:
        assert host_bytes == new
        # No leftover content from the stale image.
        assert b'STALE-OLD-IMAGE' not in host_bytes
+        assert sorted(os.listdir(stale_dir)) == ['image_1.png']

    @pytest.mark.asyncio
    async def test_inbound_host_replaces_query_symlink_without_touching_other_workspace(self, tmp_path):
@@ -137,10 +137,7 @@ async def test_officialaccount_message_converter_maps_components_to_passive_text
        )
    )

-    assert 'hi' in content
-    assert '[Image]' in content
-    assert '[File: a.txt]' in content
-    assert 'quoted' in content
+    assert content == 'hi\n[Image]\n[File: a.txt]\nquoted'


@pytest.mark.asyncio
@@ -11,12 +11,20 @@ from langbot.libs.openclaw_weixin_api.client import (
 from langbot.libs.openclaw_weixin_api.types import ApiError


-def test_cdn_crypto_helpers_round_trip():
-    original = b'tenant-media' * 128
+def test_cdn_crypto_helpers_round_trip(monkeypatch):
+    # AES-128-ECB with PKCS7 padding, independently generated using OpenSSL.
+    key = bytes.fromhex('2b7e151628aed2a6abf7158809cf4f3c')
+    original = bytes.fromhex('6bc1bee22e409f96e93d7e117393172a')
+    expected_ciphertext = bytes.fromhex('3ad77bb40d7a3660a89ecaf32466ef97a254be88e037ddd9d79fb6411c3f9df8')
+    monkeypatch.setattr('langbot.libs.openclaw_weixin_api.client.os.urandom', lambda size: key)

-    aes_key_hex, _encoded_key, encrypted, _raw_md5 = _encrypt_cdn_payload(original)
+    aes_key_hex, encoded_key, encrypted, raw_md5 = _encrypt_cdn_payload(original)

-    assert _decrypt_cdn_payload(encrypted, bytes.fromhex(aes_key_hex)) == original
+    assert aes_key_hex == '2b7e151628aed2a6abf7158809cf4f3c'
+    assert encoded_key == 'MmI3ZTE1MTYyOGFlZDJhNmFiZjcxNTg4MDljZjRmM2M='
+    assert encrypted == expected_ciphertext
+    assert raw_md5 == 'f2ecfb66e72f88f217e659ac75b1075f'
+    assert _decrypt_cdn_payload(expected_ciphertext, key) == original


@pytest.mark.asyncio
@@ -164,12 +164,7 @@ async def test_slack_message_converter_maps_common_components_to_text():
        )
    )

-    assert 'hi' in text
-    assert '<@U-2>' in text
-    assert '<!channel>' in text
-    assert 'https://example.test/a.png' in text
-    assert 'a.txt' in text
-    assert 'quoted' in text
+    assert text == 'hi\n<@U-2>\n<!channel>\nhttps://example.test/a.png\na.txt\nquoted'


@pytest.mark.asyncio
@@ -604,6 +604,17 @@ async def test_telegram_platform_apis_call_underlying_bot_methods():
    assert invite['invite_link'] == 'https://t.me/+abc'
    assert await PLATFORM_API_MAP['answer_callback_query'](bot, {'callback_query_id': 'cb'}) == {'ok': True}

+    bot.pin_chat_message.assert_awaited_once_with(chat_id=1, message_id=2, disable_notification=False)
+    bot.unpin_chat_message.assert_awaited_once_with(chat_id=1, message_id=2)
+    bot.unpin_all_chat_messages.assert_awaited_once_with(chat_id=1)
+    bot.get_chat_administrators.assert_awaited_once_with(chat_id=1)
+    bot.set_chat_title.assert_awaited_once_with(chat_id=1, title='New')
+    bot.set_chat_description.assert_awaited_once_with(chat_id=1, description='Desc')
+    bot.get_chat_member_count.assert_awaited_once_with(chat_id=1)
+    bot.send_chat_action.assert_awaited_once_with(chat_id=1, action='typing')
+    bot.create_chat_invite_link.assert_awaited_once_with(chat_id=1, name='invite', expire_date=None, member_limit=None)
+    bot.answer_callback_query.assert_awaited_once_with(callback_query_id='cb', text=None, show_alert=False, url=None)
+

@pytest.mark.asyncio
 async def test_telegram_unmute_member_uses_current_chat_permissions_fields():
@@ -90,17 +90,23 @@ async def test_image_defaults_to_png():
    adapter, _, _ = _make_adapter()
    chain = [{'type': 'Image', 'path': f'{_UPLOAD_PREFIX}blob'}]
    await adapter._process_image_components(_make_connection(), chain)
-    assert chain[0]['base64'].startswith('data:image/png;base64,')
+    assert chain[0]['base64'] == 'data:image/png;base64,aGVsbG8='


@pytest.mark.asyncio
-async def test_voice_uses_guessed_or_wav_mimetype():
+async def test_voice_uses_guessed_or_wav_mimetype(monkeypatch):
    adapter, storage_mgr, _ = _make_adapter()
-    chain = [{'type': 'Voice', 'path': f'{_UPLOAD_PREFIX}clip.wav'}]
+    path = f'{_UPLOAD_PREFIX}clip.unknown'
+    chain = [{'type': 'Voice', 'path': path}]
+    guess_type = Mock(return_value=(None, None))
+    monkeypatch.setattr('mimetypes.guess_type', guess_type)
+
    await adapter._process_image_components(_make_connection(), chain)
-    assert chain[0]['base64'].startswith('data:audio/')
+
+    guess_type.assert_called_once_with(path)
+    assert chain[0]['base64'] == 'data:audio/wav;base64,aGVsbG8='
    assert chain[0]['path'] == ''
-    storage_mgr.delete_scoped_object_key.assert_awaited_once()
+    storage_mgr.delete_scoped_object_key.assert_awaited_once_with(_CONTEXT, path, expected_owner_type='upload_image')


@pytest.mark.asyncio