feat(cloud): harden multi-tenant runtime resources

tests/unit_tests/core/test_app_shutdown.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from langbot.pkg.core.app import Application
+
+
+@pytest.mark.asyncio
+async def test_shutdown_closes_mcp_session_manager_once() -> None:
+    app = Application()
+    stop_session_manager = AsyncMock()
+    app.platform_mgr = SimpleNamespace(shutdown=AsyncMock())
+    app.tool_mgr = SimpleNamespace(shutdown=AsyncMock())
+    app.model_mgr = SimpleNamespace(shutdown=AsyncMock())
+    app.box_service = SimpleNamespace(shutdown=AsyncMock())
+    app.plugin_connector = SimpleNamespace(aclose=AsyncMock())
+    app.telemetry = SimpleNamespace(shutdown=AsyncMock())
+    app.vector_db_mgr = SimpleNamespace(shutdown=AsyncMock())
+    app.storage_mgr = SimpleNamespace(shutdown=AsyncMock())
+    manifest_provider = SimpleNamespace(aclose=AsyncMock())
+    app.deployment = SimpleNamespace(manifest_provider=manifest_provider)
+    persistence_engine = SimpleNamespace(dispose=AsyncMock())
+    app.persistence_mgr = SimpleNamespace(db=SimpleNamespace(engine=persistence_engine))
+    app.http_ctrl = SimpleNamespace(mcp_mount=SimpleNamespace(stop_session_manager=stop_session_manager))
+
+    await app.shutdown()
+    await app.shutdown()
+
+    stop_session_manager.assert_awaited_once()
+    app.platform_mgr.shutdown.assert_awaited_once()
+    app.tool_mgr.shutdown.assert_awaited_once()
+    app.model_mgr.shutdown.assert_awaited_once()
+    app.box_service.shutdown.assert_awaited_once()
+    app.plugin_connector.aclose.assert_awaited_once()
+    app.telemetry.shutdown.assert_awaited_once()
+    app.vector_db_mgr.shutdown.assert_awaited_once()
+    app.storage_mgr.shutdown.assert_awaited_once()
+    manifest_provider.aclose.assert_awaited_once()
+    persistence_engine.dispose.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_dispose_tracks_only_one_shutdown_task() -> None:
+    app = Application()
+    app.event_loop = asyncio.get_running_loop()
+
+    app.dispose()
+    shutdown_task = app._shutdown_task
+    app.dispose()
+
+    assert shutdown_task is not None
+    assert app._shutdown_task is shutdown_task
+    await shutdown_task
+
+    app.dispose()
+    assert app._shutdown_task is shutdown_task
+
+
+@pytest.mark.asyncio
+async def test_runtime_resource_stats_are_aggregate_and_constant_time() -> None:
+    app = Application()
+    app.event_loop = asyncio.get_running_loop()
+    app.blocking_executor = SimpleNamespace(
+        snapshot=lambda: {
+            'inflight': 3,
+            'running': 2,
+            'pending': 1,
+            'rejected_total': 4,
+        }
+    )
+    app.task_mgr = SimpleNamespace(get_stats=lambda: {'total': 5, 'completed': 2})
+    app.query_pool = SimpleNamespace(
+        queries=[object()],
+        cached_queries={},
+        active_query_count_by_workspace={'workspace-a': 1},
+    )
+    app.model_mgr = SimpleNamespace(
+        provider_dict={'provider': object()},
+        llm_model_dict={},
+        embedding_model_dict={},
+        rerank_model_dict={},
+    )
+    app.platform_mgr = SimpleNamespace(_bots_by_key={})
+    app.pipeline_mgr = SimpleNamespace(_pipelines_by_key={})
+    app.rag_mgr = SimpleNamespace(knowledge_bases={})
+    app.plugin_connector = SimpleNamespace(_known_desired_states={'installation': object()})
+    app.tool_mgr = SimpleNamespace(
+        mcp_tool_loader=SimpleNamespace(
+            _sessions={},
+            _hosted_mcp_tasks=[],
+            _host_dispatch_tasks=set(),
+        )
+    )
+    app.telemetry = SimpleNamespace(send_tasks=[])
+
+    stats = app.get_runtime_resource_stats()
+
+    assert stats['asyncio_tasks'] >= 1
+    assert stats['event_loop'] == {
+        'running': False,
+        'samples_total': 0,
+        'last_lag_ms': 0,
+        'recent_p95_lag_ms': 0,
+        'recent_max_lag_ms': 0,
+        'max_lag_ms': 0,
+    }
+    assert stats['blocking_executor']['rejected_total'] == 4
+    assert stats['application_tasks'] == {
+        'total': 5,
+        'completed': 2,
+    }
+    assert stats['query_pool'] == {
+        'queued': 1,
+        'cached': 0,
+        'active_workspaces': 1,
+    }
+    assert stats['models']['providers'] == 1
+    assert stats['runtimes']['plugin_installations'] == 1

tests/unit_tests/core/test_boot.py

@@ -2,13 +2,37 @@ from __future__ import annotations
 
 import signal
 from types import SimpleNamespace
-from unittest.mock import Mock
+from unittest.mock import AsyncMock, Mock
 
 import pytest
 
 from langbot.pkg.core import boot
 
 
+@pytest.mark.asyncio
+async def test_make_app_shuts_down_partially_built_application(monkeypatch):
+    app_inst = SimpleNamespace(
+        event_loop=None,
+        shutdown=AsyncMock(),
+        initialize=AsyncMock(),
+    )
+
+    class FailingStage:
+        async def run(self, ap):
+            assert ap is app_inst
+            raise RuntimeError('startup failed')
+
+    monkeypatch.setattr(boot.app, 'Application', lambda: app_inst)
+    monkeypatch.setattr(boot, 'stage_order', ['FailingStage'])
+    monkeypatch.setitem(boot.stage.preregistered_stages, 'FailingStage', FailingStage)
+
+    with pytest.raises(RuntimeError, match='startup failed'):
+        await boot.make_app(SimpleNamespace())
+
+    app_inst.shutdown.assert_awaited_once()
+    app_inst.initialize.assert_not_awaited()
+
+
 @pytest.mark.asyncio
 async def test_main_signal_handler_handles_sigint_before_app_created(monkeypatch):
     captured_handler = {}

tests/unit_tests/core/test_load_config.py

@@ -333,11 +333,23 @@ class TestApplyEnvOverridesToConfig:
             {
                 'PLUGIN__WORKER__MAX_MEMORY_MB': '768',
                 'MCP__STDIO__ENABLED': 'false',
+                'SYSTEM__BLOCKING_EXECUTOR__MAX_WORKERS': '12',
+                'SYSTEM__BLOCKING_EXECUTOR__MAX_PENDING': '256',
+                'SYSTEM__BLOCKING_EXECUTOR__MAX_INFLIGHT_PER_SCOPE': '3',
             },
             clear=True,
         ):
             result = load_config._apply_env_overrides_to_config(completed)
 
+        assert result['system']['blocking_executor'] == {
+            'max_workers': 12,
+            'max_pending': 256,
+            'max_inflight_per_scope': 3,
+        }
+        assert isinstance(
+            result['system']['blocking_executor']['max_workers'],
+            int,
+        )
         assert result['plugin']['worker']['max_memory_mb'] == 768
         assert isinstance(result['plugin']['worker']['max_memory_mb'], int)
         assert result['mcp']['stdio']['enabled'] is False

tests/unit_tests/core/test_taskmgr.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 import pytest
 import asyncio
 import contextvars
+import inspect
 import sys
 from unittest.mock import Mock, MagicMock
 from contextlib import contextmanager
@@ -265,6 +266,28 @@ class TestTaskWrapper:
 
         wrapper.cancel()
 
+    @pytest.mark.asyncio
+    async def test_workspace_task_sets_blocking_work_scope(self):
+        """Detached tasks recover tenant fairness from durable ownership."""
+        _, TaskWrapper, _ = get_taskmgr_classes()
+        from langbot.pkg.utils.bounded_executor import (
+            current_blocking_work_scope,
+        )
+
+        mock_app = create_mock_app()
+
+        async def read_scope():
+            return current_blocking_work_scope()
+
+        wrapper = TaskWrapper(
+            mock_app,
+            read_scope(),
+            workspace_uuid='workspace-a',
+        )
+
+        assert await wrapper.task == 'workspace-a'
+        assert current_blocking_work_scope() is None
+
     @pytest.mark.asyncio
     async def test_to_dict_serialization(self):
         """Test TaskWrapper.to_dict serialization."""
@@ -530,6 +553,56 @@ class TestAsyncTaskManager:
 
         wrapper.cancel()
 
+    @pytest.mark.asyncio
+    async def test_create_user_task_enforces_workspace_active_limit_and_closes_rejected_coroutine(self):
+        """A noisy Workspace cannot accumulate unbounded background work."""
+        _, _, AsyncTaskManager = get_taskmgr_classes()
+        mock_app = create_mock_app()
+        mock_app.instance_config.data['system']['task_retention'].update(
+            {
+                'max_active_user_tasks': 10,
+                'max_active_user_tasks_per_workspace': 1,
+            }
+        )
+        manager = AsyncTaskManager(mock_app)
+
+        async def long_coro():
+            await asyncio.sleep(10)
+
+        first = manager.create_user_task(long_coro(), workspace_uuid='workspace-a')
+        rejected = long_coro()
+        with pytest.raises(RuntimeError, match='Workspace has too many active user operations'):
+            manager.create_user_task(rejected, workspace_uuid='workspace-a')
+
+        assert inspect.getcoroutinestate(rejected) == inspect.CORO_CLOSED
+        other_workspace = manager.create_user_task(long_coro(), workspace_uuid='workspace-b')
+        first.cancel()
+        other_workspace.cancel()
+
+    @pytest.mark.asyncio
+    async def test_create_user_task_enforces_instance_active_limit(self):
+        """The shared process retains a hard cap even across Workspaces."""
+        _, _, AsyncTaskManager = get_taskmgr_classes()
+        mock_app = create_mock_app()
+        mock_app.instance_config.data['system']['task_retention'].update(
+            {
+                'max_active_user_tasks': 1,
+                'max_active_user_tasks_per_workspace': 10,
+            }
+        )
+        manager = AsyncTaskManager(mock_app)
+
+        async def long_coro():
+            await asyncio.sleep(10)
+
+        first = manager.create_user_task(long_coro(), workspace_uuid='workspace-a')
+        rejected = long_coro()
+        with pytest.raises(RuntimeError, match='instance has too many active user operations'):
+            manager.create_user_task(rejected, workspace_uuid='workspace-b')
+
+        assert inspect.getcoroutinestate(rejected) == inspect.CORO_CLOSED
+        first.cancel()
+
     @pytest.mark.asyncio
     async def test_get_task_by_id(self):
         """Test get_task_by_id returns correct task."""