feat(cloud): harden multi-tenant runtime resources

tests/unit_tests/api/http/test_bounded_json_request.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import quart
+
+from langbot.pkg.api.http.controller import main as controller_main
+from langbot.pkg.utils import bounded_executor
+
+
+async def test_bounded_json_request_decodes_off_loop_in_workspace_scope(
+    monkeypatch,
+):
+    app = quart.Quart(__name__)
+    app.request_class = controller_main.BoundedJSONRequest
+    observed_scopes: list[str | None] = []
+
+    async def fake_to_thread(fn, *args, **kwargs):
+        observed_scopes.append(bounded_executor.current_blocking_work_scope())
+        return fn(*args, **kwargs)
+
+    monkeypatch.setattr(
+        controller_main.asyncio,
+        'to_thread',
+        fake_to_thread,
+    )
+
+    @app.post('/json')
+    async def parse_json():
+        with bounded_executor.blocking_work_scope('workspace-a'):
+            payload = await quart.request.get_json()
+        return quart.jsonify(payload)
+
+    response = await app.test_client().post(
+        '/json',
+        json={'nested': {'value': 1}},
+    )
+
+    assert response.status_code == 200
+    assert await response.get_json() == {'nested': {'value': 1}}
+    assert observed_scopes == ['workspace-a']

tests/unit_tests/api/http/test_internal_error_responses.py

@@ -9,6 +9,10 @@ import quart
 
 from langbot.pkg.api.http.controller import group
 from langbot.pkg.api.http.controller.groups.webhooks import WebhookRouterGroup
+from langbot.pkg.utils.bounded_executor import (
+    BlockingWorkCapacityError,
+    current_blocking_work_scope,
+)
 
 
 pytestmark = pytest.mark.asyncio
@@ -34,6 +38,16 @@ class _AuthenticatedRouterGroup(group.RouterGroup):
             return self.success()
 
 
+class _BlockingCapacityRouterGroup(group.RouterGroup):
+    name = 'blocking-capacity-test'
+    path = '/blocking-capacity-test'
+
+    async def initialize(self) -> None:
+        @self.route('', methods=['GET'], auth_type=group.AuthType.NONE)
+        async def _():
+            raise BlockingWorkCapacityError('Workspace blocking executor capacity reached')
+
+
 class _InvalidAccountRouterGroup(group.RouterGroup):
     name = 'invalid-account-test'
     path = '/invalid-account-test'
@@ -102,6 +116,20 @@ async def test_public_webhook_error_uses_same_generic_error_contract():
     assert 'do-not-return' not in (await response.get_data(as_text=True))
 
 
+async def test_blocking_work_capacity_maps_to_retryable_http_response():
+    application = SimpleNamespace(logger=Mock())
+    quart_app = quart.Quart(__name__)
+    await _BlockingCapacityRouterGroup(application, quart_app).initialize()
+
+    response = await quart_app.test_client().get('/blocking-capacity-test')
+
+    assert response.status_code == 429
+    assert await response.get_json() == {
+        'code': 'blocking_work_capacity_exceeded',
+        'msg': 'Workspace blocking executor capacity reached',
+    }
+
+
 async def test_public_webhook_carries_scope_without_holding_database_session():
     class ScopeOnlyPersistenceManager:
         mode = SimpleNamespace(value='cloud_runtime')
@@ -128,6 +156,7 @@ async def test_public_webhook_carries_scope_without_holding_database_session():
         async def handle_unified_webhook(self, **_kwargs):
             assert persistence_mgr.active_workspace == workspace_uuid
             assert persistence_mgr.current_session() is None
+            assert current_blocking_work_scope() == workspace_uuid
             return {'ok': True}
 
     async def get_execution_binding(resolved_workspace_uuid, expected_generation=None):
@@ -156,6 +185,41 @@ async def test_public_webhook_carries_scope_without_holding_database_session():
     assert persistence_mgr.active_workspace is None
 
 
+async def test_public_webhook_blocking_capacity_is_retryable():
+    workspace_uuid = '00000000-0000-0000-0000-00000000000a'
+    bot_uuid = '11111111-1111-4111-8111-111111111111'
+
+    class Adapter:
+        async def handle_unified_webhook(self, **_kwargs):
+            raise BlockingWorkCapacityError(
+                'Workspace blocking executor capacity reached',
+                scope=workspace_uuid,
+            )
+
+    runtime_bot = SimpleNamespace(
+        workspace_uuid=workspace_uuid,
+        placement_generation=4,
+        enable=True,
+        adapter=Adapter(),
+    )
+    application = SimpleNamespace(
+        logger=Mock(),
+        persistence_mgr=SimpleNamespace(mode=SimpleNamespace(value='oss')),
+        platform_mgr=SimpleNamespace(resolve_public_bot=AsyncMock(return_value=runtime_bot)),
+        workspace_service=SimpleNamespace(get_execution_binding=AsyncMock(return_value=None)),
+    )
+    quart_app = quart.Quart(__name__)
+    await WebhookRouterGroup(application, quart_app).initialize()
+
+    response = await quart_app.test_client().post(f'/bots/{bot_uuid}')
+
+    assert response.status_code == 429
+    assert await response.get_json() == {
+        'code': 'blocking_work_capacity_exceeded',
+        'msg': 'Workspace blocking executor capacity reached',
+    }
+
+
 async def test_authentication_failure_does_not_return_internal_exception_text():
     logger = Mock()
     application = SimpleNamespace(

tests/unit_tests/api/service/test_mcp_service.py

@@ -29,6 +29,7 @@ from langbot.pkg.api.http.context import (
     WorkspaceContext,
 )
 from langbot.pkg.api.http.service.mcp import MCPService, redact_mcp_secrets, restore_mcp_secret_placeholders
+from langbot.pkg.core.taskmgr import TaskCapacityError
 from langbot.pkg.entity.persistence.mcp import MCPServer
 from langbot.pkg.provider.tools.loaders.mcp_policy import MCPStdioDisabledError
 from langbot.pkg.workspace.errors import WorkspaceNotFoundError
@@ -1050,3 +1051,23 @@ class TestMCPServiceTestMCPServer:
         # Verify - load_mcp_server called
         ap.tool_mgr.mcp_tool_loader.load_mcp_server.assert_called_once()
         assert task_id == 456
+
+    async def test_rejected_transient_test_session_is_shut_down(self):
+        ap = SimpleNamespace()
+        mock_session = MagicMock()
+        mock_session.shutdown = AsyncMock()
+        ap.tool_mgr = SimpleNamespace(
+            mcp_tool_loader=SimpleNamespace(load_mcp_server=AsyncMock(return_value=mock_session))
+        )
+
+        def reject(coroutine, **_kwargs):
+            coroutine.close()
+            raise TaskCapacityError('capacity')
+
+        ap.task_mgr = SimpleNamespace(create_user_task=Mock(side_effect=reject))
+        service = _service(ap)
+
+        with pytest.raises(TaskCapacityError, match='capacity'):
+            await service.test_mcp_server(_CONTEXT, '_', {'name': 'New Server'})
+
+        mock_session.shutdown.assert_awaited_once_with()

tests/unit_tests/api/service/test_space_service.py

@@ -13,6 +13,8 @@ Source: src/langbot/pkg/api/http/service/space.py
 
 from __future__ import annotations
 
+from collections import OrderedDict
+import json
 from urllib.parse import parse_qs, urlsplit
 
 import pytest
@@ -28,6 +30,23 @@ from langbot.pkg.entity.persistence.user import User
 pytestmark = pytest.mark.asyncio
 
 
+def _set_response_body(response: MagicMock, body: dict | str) -> None:
+    """Configure an aiohttp-like streaming body on an HTTP response mock."""
+
+    raw_body = body.encode() if isinstance(body, str) else json.dumps(body).encode()
+
+    class Content:
+        async def iter_chunked(self, _chunk_size: int):
+            midpoint = max(len(raw_body) // 2, 1)
+            yield raw_body[:midpoint]
+            if midpoint < len(raw_body):
+                yield raw_body[midpoint:]
+
+    response.headers = {}
+    response.content = Content()
+    response.charset = 'utf-8'
+
+
 def _create_mock_user(
     email: str = 'test@example.com',
     account_type: str = 'space',
@@ -292,6 +311,40 @@ class TestSpaceServiceGetCredits:
         # Verify - returns cached value without API call
         assert result == 100
 
+    async def test_cached_credit_lookup_does_not_scan_all_users(self):
+        ap = SimpleNamespace()
+        ap.instance_config = SimpleNamespace(data={})
+        ap.persistence_mgr = SimpleNamespace()
+        service = SpaceService(ap)
+
+        class AtMostOneStepOrderedDict(OrderedDict):
+            def __iter__(self):
+                iterator = super().__iter__()
+                yielded = False
+
+                def next_entry():
+                    nonlocal yielded
+                    if yielded:
+                        raise AssertionError('credits cache scanned all users')
+                    yielded = True
+                    return next(iterator)
+
+                class AtMostOneStepIterator:
+                    def __iter__(self):
+                        return self
+
+                    def __next__(self):
+                        return next_entry()
+
+                return AtMostOneStepIterator()
+
+        now = time.time()
+        service._credits_cache = AtMostOneStepOrderedDict(
+            (f'user-{index}@example.com', (index, now)) for index in range(512)
+        )
+
+        assert await service.get_credits('user-511@example.com') == 511
+
     async def test_get_credits_cache_expired_refreshes(self):
         """Refreshes expired cache."""
         # Setup
@@ -406,6 +459,7 @@ class TestSpaceServiceRefreshToken:
                 },
             }
         )
+        _set_response_body(mock_response, mock_response.json.return_value)
 
         with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
             mock_session_obj = MagicMock()
@@ -441,6 +495,7 @@ class TestSpaceServiceRefreshToken:
             }
         )
         mock_response.text = AsyncMock(return_value='{"code":1,"msg":"Invalid refresh token"}')
+        _set_response_body(mock_response, mock_response.json.return_value)
 
         with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
             mock_session_obj = MagicMock()
@@ -467,6 +522,7 @@ class TestSpaceServiceRefreshToken:
         mock_response = MagicMock()
         mock_response.status = 500
         mock_response.text = AsyncMock(return_value='Internal Server Error')
+        _set_response_body(mock_response, mock_response.text.return_value)
 
         with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
             mock_session_obj = MagicMock()
@@ -506,6 +562,7 @@ class TestSpaceServiceExchangeOAuthCode:
                 },
             }
         )
+        _set_response_body(mock_response, mock_response.json.return_value)
 
         with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
             mock_session_obj = MagicMock()
@@ -535,6 +592,7 @@ class TestSpaceServiceExchangeOAuthCode:
         mock_response.status = 200
         mock_response.json = AsyncMock(return_value={'code': 1, 'msg': 'Invalid code'})
         mock_response.text = AsyncMock(return_value='{"code":1,"msg":"Invalid code"}')
+        _set_response_body(mock_response, mock_response.json.return_value)
 
         with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
             mock_session_obj = MagicMock()
@@ -573,6 +631,7 @@ class TestSpaceServiceGetUserInfoRaw:
                 },
             }
         )
+        _set_response_body(mock_response, mock_response.json.return_value)
 
         with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
             mock_session_obj = MagicMock()
@@ -603,6 +662,7 @@ class TestSpaceServiceGetUserInfoRaw:
         mock_response.status = 200
         mock_response.json = AsyncMock(return_value={'code': 1, 'msg': 'Unauthorized'})
         mock_response.text = AsyncMock(return_value='{"code":1,"msg":"Unauthorized"}')
+        _set_response_body(mock_response, mock_response.json.return_value)
 
         with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
             mock_session_obj = MagicMock()
@@ -703,6 +763,7 @@ class TestSpaceServiceGetModels:
                 },
             }
         )
+        _set_response_body(mock_response, mock_response.json.return_value)
 
         with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
             mock_session_obj = MagicMock()
@@ -732,6 +793,7 @@ class TestSpaceServiceGetModels:
         mock_response.status = 200
         mock_response.json = AsyncMock(return_value={'code': 1, 'msg': 'Unauthorized'})
         mock_response.text = AsyncMock(return_value='{"code":1,"msg":"Unauthorized"}')
+        _set_response_body(mock_response, mock_response.json.return_value)
 
         with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
             mock_session_obj = MagicMock()

tests/unit_tests/api/service/test_user_service.py

@@ -29,11 +29,25 @@ from langbot.pkg.entity.errors.account import (
     SpaceAccountBindingRequiredError,
     SpaceAccountNotRegisteredError,
 )
+from langbot.pkg.utils.bounded_executor import BlockingWorkCapacityError
 
 
 pytestmark = pytest.mark.asyncio
 
 
+async def test_password_hashing_rejects_concurrent_waiters() -> None:
+    service = UserService(SimpleNamespace())
+    await service._password_hash_lock.acquire()
+    try:
+        with pytest.raises(
+            BlockingWorkCapacityError,
+            match='Password hashing capacity reached',
+        ):
+            await service._hash_password('secret')
+    finally:
+        service._password_hash_lock.release()
+
+
 class TestSpaceOAuthState:
     async def test_login_state_is_opaque_single_use(self):
         service = UserService(SimpleNamespace())
@@ -91,6 +105,32 @@ class TestSpaceOAuthState:
         assert consumed.account is None
         assert consumed.launch_workspace_uuid == 'workspace-a'
 
+    async def test_issue_state_does_not_scan_all_live_states(self, monkeypatch):
+        service = UserService(SimpleNamespace())
+        for _ in range(512):
+            await service.issue_space_oauth_state('login')
+
+        class NoGlobalIterationDict(dict):
+            def __iter__(self):
+                raise AssertionError('OAuth state issuance scanned all live states')
+
+            def keys(self):
+                raise AssertionError('OAuth state issuance scanned all live states')
+
+            def items(self):
+                raise AssertionError('OAuth state issuance scanned all live states')
+
+            def values(self):
+                raise AssertionError('OAuth state issuance scanned all live states')
+
+        guarded_states = NoGlobalIterationDict(service._space_oauth_states)
+        monkeypatch.setattr(service, '_space_oauth_states', guarded_states)
+
+        state = await service.issue_space_oauth_state('login')
+
+        assert await service.consume_space_oauth_state(state, 'login') is None
+        assert len(guarded_states) == 512
+
 
 def _create_mock_user(
     email: str = 'test@example.com',

tests/unit_tests/api/test_adapter_session_scoping.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import asyncio
 from types import SimpleNamespace
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, Mock
 
 import lark_oapi
 import pytest
@@ -19,7 +19,9 @@ from langbot.pkg.api.http.controller.groups.platform.adapters import (
     _AdapterSessionScope,
     _bind_session_scope,
     _get_owned_session,
+    _make_room_for_session,
     _pop_owned_session,
+    _start_adapter_session_task,
 )
 
 
@@ -94,6 +96,10 @@ async def _create_client(*, role: str = 'developer'):
             ),
         )
 
+    class TestTaskManager:
+        def create_user_task(self, coro, **_kwargs):
+            return SimpleNamespace(task=asyncio.create_task(coro))
+
     application = SimpleNamespace(
         user_service=SimpleNamespace(
             get_authenticated_account=AsyncMock(side_effect=get_authenticated_account),
@@ -102,6 +108,7 @@ async def _create_client(*, role: str = 'developer'):
             resolve_account_workspace=AsyncMock(side_effect=resolve_account_workspace),
         ),
         platform_mgr=SimpleNamespace(),
+        task_mgr=TestTaskManager(),
     )
     router = AdaptersRouterGroup(application, quart_app)
     await router.initialize()
@@ -142,6 +149,58 @@ async def test_session_scope_matches_exact_tenant_placement_and_principal():
     assert sessions == {}
 
 
+async def test_session_capacity_evicts_oldest_session_in_same_workspace():
+    owner_context = _request_context()
+    sessions: dict[str, dict] = {}
+    tasks = []
+    for index in range(10):
+        task = SimpleNamespace(done=Mock(return_value=False), cancel=Mock())
+        tasks.append(task)
+        session = {'created_at': float(index), 'task': task}
+        _bind_session_scope(session, owner_context)
+        sessions[f'session-{index}'] = session
+
+    _make_room_for_session(sessions, owner_context)
+
+    assert 'session-0' not in sessions
+    assert len(sessions) == 9
+    tasks[0].cancel.assert_called_once_with()
+
+
+async def test_adapter_session_task_uses_tenant_task_admission():
+    blocker = asyncio.Event()
+
+    async def credential_exchange():
+        await blocker.wait()
+
+    task_manager = SimpleNamespace(create_user_task=Mock())
+
+    def create_user_task(coro, **_kwargs):
+        return SimpleNamespace(task=asyncio.create_task(coro))
+
+    task_manager.create_user_task.side_effect = create_user_task
+    application = SimpleNamespace(task_mgr=task_manager)
+    request_context = _request_context()
+
+    returned = _start_adapter_session_task(
+        application,
+        credential_exchange(),
+        adapter='lark',
+        session_id='session-test',
+        request_context=request_context,
+    )
+
+    assert returned is not None
+    task_manager.create_user_task.assert_called_once()
+    kwargs = task_manager.create_user_task.call_args.kwargs
+    assert kwargs['kind'] == 'platform-adapter-credential-exchange'
+    assert kwargs['instance_uuid'] == request_context.instance_uuid
+    assert kwargs['workspace_uuid'] == request_context.workspace_uuid
+    assert kwargs['placement_generation'] == request_context.placement_generation
+    blocker.set()
+    await returned
+
+
 async def test_lark_session_status_and_delete_hide_cross_scope_sessions(monkeypatch):
     registration_blocker = asyncio.Event()
 

tests/unit_tests/api/test_websocket_chat_uow.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 from contextlib import asynccontextmanager
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, Mock
@@ -13,6 +14,11 @@ from langbot.pkg.api.http.context import (
     WorkspaceContext,
 )
 from langbot.pkg.api.http.controller.groups.pipelines.websocket_chat import WebSocketChatRouterGroup
+from langbot.pkg.api.http.controller.groups.pipelines.websocket_chat import (
+    create_scoped_duplex_tasks,
+)
+from langbot.pkg.api.http.controller.groups.pipelines.websocket_chat import wait_for_duplex_tasks
+from langbot.pkg.utils.bounded_executor import current_blocking_work_scope
 
 
 @pytest.mark.asyncio
@@ -67,3 +73,72 @@ async def test_websocket_pipeline_lookup_opens_workspace_uow_after_auth_scope_cl
 
     assert result is adapter
     assert scopes == [workspace_uuid]
+
+
+@pytest.mark.asyncio
+async def test_duplex_websocket_tasks_cancel_blocked_peer_when_one_direction_ends() -> None:
+    blocked = asyncio.Event()
+
+    async def receive_forever() -> None:
+        blocked.set()
+        await asyncio.Future()
+
+    async def send_finishes() -> None:
+        await blocked.wait()
+
+    receive_task = asyncio.create_task(receive_forever())
+    send_task = asyncio.create_task(send_finishes())
+
+    await asyncio.wait_for(
+        wait_for_duplex_tasks(receive_task, send_task),
+        timeout=1,
+    )
+
+    assert receive_task.cancelled()
+    assert send_task.done()
+
+
+@pytest.mark.asyncio
+async def test_duplex_websocket_tasks_allow_terminal_send_to_drain() -> None:
+    receive_finished = asyncio.Event()
+    send_drained = asyncio.Event()
+
+    async def receive_finishes() -> None:
+        receive_finished.set()
+
+    async def send_terminal_frame() -> None:
+        await receive_finished.wait()
+        await asyncio.sleep(0)
+        send_drained.set()
+
+    receive_task = asyncio.create_task(receive_finishes())
+    send_task = asyncio.create_task(send_terminal_frame())
+
+    await wait_for_duplex_tasks(receive_task, send_task)
+
+    assert send_drained.is_set()
+    assert send_task.done()
+    assert not send_task.cancelled()
+
+
+@pytest.mark.asyncio
+async def test_duplex_websocket_tasks_share_trusted_workspace_budget() -> None:
+    observed: list[tuple[str, str | None]] = []
+
+    async def observe(direction: str) -> None:
+        await asyncio.sleep(0)
+        observed.append((direction, current_blocking_work_scope()))
+
+    receive_task, send_task = create_scoped_duplex_tasks(
+        observe('receive'),
+        observe('send'),
+        'workspace-a',
+    )
+
+    await asyncio.gather(receive_task, send_task)
+
+    assert sorted(observed) == [
+        ('receive', 'workspace-a'),
+        ('send', 'workspace-a'),
+    ]
+    assert current_blocking_work_scope() is None