feat(cloud): harden multi-tenant runtime resources

tests/unit_tests/pipeline/test_chat_handler.py

@@ -464,3 +464,13 @@ class TestChatHandlerHelper:
         handler = chat.ChatMessageHandler(fake_app)
         result = handler.cut_str('first line\nsecond line')
         assert '...' in result
+
+    def test_response_size_limit_uses_instance_config(self, fake_app):
+        from langbot_plugin.api.entities.builtin.provider.message import Message
+
+        fake_app.instance_config.data['system'] = {'response_limits': {'max_generated_chars': 4}}
+        chat = get_chat_handler()
+        handler = chat.ChatMessageHandler(fake_app)
+
+        with pytest.raises(RuntimeError, match='configured limit'):
+            handler._check_response_size(Message(role='assistant', content='12345'))

tests/unit_tests/pipeline/test_controller_tenancy.py

@@ -36,6 +36,7 @@ async def test_controller_drops_stale_query_before_pipeline_lookup(
     query_pool, session = _prepare_scheduler(mock_app)
     mock_app.workspace_service.get_execution_binding.side_effect = WorkspaceGenerationMismatchError('stale generation')
     controller = Controller(mock_app)
+    initial_slots = controller.semaphore._value
 
     await controller._process_query(sample_query)
 
@@ -47,6 +48,7 @@ async def test_controller_drops_stale_query_before_pipeline_lookup(
     query_pool.remove_query.assert_awaited_once_with(sample_query)
     session._semaphore.release.assert_called_once_with()
     query_pool.condition.notify_all.assert_called_once_with()
+    assert controller.semaphore._value == initial_slots
 
 
 @pytest.mark.asyncio

tests/unit_tests/pipeline/test_longtext_image.py

@@ -0,0 +1,17 @@
+from unittest.mock import Mock
+
+from langbot.pkg.pipeline.longtext.strategies.image import Text2ImageStrategy
+
+
+class _WideFont:
+    def getlength(self, text: str) -> int:
+        return len(text) * 100
+
+
+def test_image_strategy_line_split_always_consumes_input():
+    strategy = Text2ImageStrategy(Mock())
+
+    lines = strategy._split_text_lines('abc', 1, _WideFont())
+
+    assert lines == ['a', 'b', 'c']
+    assert ''.join(lines) == 'abc'

tests/unit_tests/pipeline/test_n8nsvapi.py

@@ -37,7 +37,11 @@ _saved_modules = {name: sys.modules.get(name) for name in _import_stubs}
 for _name, _stub in _import_stubs.items():
     sys.modules[_name] = _stub
 try:
-    from langbot.pkg.provider.runners.n8nsvapi import N8nServiceAPIRunner
+    from langbot.pkg.provider.runners.n8nsvapi import (
+        N8nAPIError,
+        N8nServiceAPIRunner,
+        _MAX_N8N_RESPONSE_CHARS,
+    )
 finally:
     for _name, _original in _saved_modules.items():
         if _original is None:
@@ -230,6 +234,17 @@ async def test_plain_json_non_dict_response():
     assert chunks[0].content == '["a", "b"]'
 
 
+@pytest.mark.asyncio
+async def test_response_size_is_bounded():
+    runner = make_runner()
+
+    with pytest.raises(N8nAPIError, match='exceeds the runtime limit'):
+        await collect_chunks(
+            runner,
+            [b'x' * (_MAX_N8N_RESPONSE_CHARS + 1)],
+        )
+
+
 @pytest.mark.asyncio
 async def test_invalid_json_returns_raw_text():
     """Non-JSON response returns raw text as-is."""

tests/unit_tests/pipeline/test_pipelinemgr.py

@@ -3,11 +3,13 @@ PipelineManager unit tests
 """
 
 import pytest
+from types import SimpleNamespace
 from unittest.mock import AsyncMock, Mock
 from importlib import import_module
 
 from langbot.pkg.api.http.context import ExecutionContext
-from langbot.pkg.workspace.errors import WorkspaceGenerationMismatchError
+from langbot.pkg.workspace.entities import WorkspaceExecutionBinding
+from langbot.pkg.workspace.errors import WorkspaceGenerationMismatchError, WorkspaceInvariantError
 
 
 def _context(pipeline_uuid: str = 'test-uuid') -> ExecutionContext:
@@ -49,6 +51,95 @@ async def test_pipeline_manager_initialize(mock_app):
     assert len(manager.pipelines) == 0
 
 
+@pytest.mark.asyncio
+async def test_cloud_startup_reuses_validated_pipeline_binding(mock_app):
+    class TenantUow:
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *_args):
+            return False
+
+    binding = WorkspaceExecutionBinding(
+        instance_uuid='test-instance',
+        workspace_uuid='test-workspace',
+        placement_generation=1,
+        write_fenced=False,
+        state='active',
+    )
+    pipeline_entity = Mock(
+        uuid='test-uuid',
+        workspace_uuid='test-workspace',
+        stages=[],
+        config={},
+        extensions_preferences={},
+    )
+    mock_app.persistence_mgr.mode = SimpleNamespace(value='cloud_runtime')
+    mock_app.persistence_mgr.tenant_uow = lambda _workspace_uuid: TenantUow()
+    mock_app.persistence_mgr.execute_async = AsyncMock(return_value=Mock(all=Mock(return_value=[pipeline_entity])))
+    mock_app.workspace_service.list_active_execution_bindings = AsyncMock(return_value=[binding])
+    mock_app.workspace_service.get_execution_binding = AsyncMock(
+        side_effect=AssertionError('startup pipeline loader repeated a validated binding lookup')
+    )
+    manager = get_pipelinemgr_module().PipelineManager(mock_app)
+    manager.stage_dict = {}
+
+    await manager.load_pipelines_from_db()
+
+    assert len(manager.pipelines) == 1
+    mock_app.workspace_service.get_execution_binding.assert_not_awaited()
+
+
+def test_generation_advance_prunes_superseded_workspace_pipelines(mock_app):
+    class NoGlobalIterationDict(dict):
+        def __iter__(self):
+            raise AssertionError('generation advance scanned every pipeline')
+
+        def items(self):
+            raise AssertionError('generation advance scanned every pipeline')
+
+        def values(self):
+            raise AssertionError('generation advance scanned every pipeline')
+
+    pipelinemgr = get_pipelinemgr_module()
+    manager = pipelinemgr.PipelineManager(mock_app)
+    old_context = _context()
+    next_context = ExecutionContext(
+        instance_uuid=old_context.instance_uuid,
+        workspace_uuid=old_context.workspace_uuid,
+        placement_generation=2,
+        pipeline_uuid=old_context.pipeline_uuid,
+    )
+    old_pipeline = SimpleNamespace(
+        execution_context=old_context,
+        workspace_uuid=old_context.workspace_uuid,
+        placement_generation=old_context.placement_generation,
+    )
+    other_pipelines = [
+        SimpleNamespace(
+            execution_context=ExecutionContext(
+                instance_uuid='test-instance',
+                workspace_uuid=f'workspace-{index}',
+                placement_generation=1,
+                pipeline_uuid=f'pipeline-{index}',
+            ),
+            workspace_uuid=f'workspace-{index}',
+            placement_generation=1,
+        )
+        for index in range(1_000)
+    ]
+    manager.pipelines = [old_pipeline, *other_pipelines]
+
+    manager._observe_execution_context(old_context)
+    manager._pipelines_by_key = NoGlobalIterationDict(manager._pipelines_by_key)
+    manager._observe_execution_context(next_context)
+    manager._pipelines_by_key = dict(manager._pipelines_by_key)
+
+    assert manager.pipelines == other_pipelines
+    with pytest.raises(WorkspaceInvariantError, match='rolled back'):
+        manager._observe_execution_context(old_context)
+
+
 @pytest.mark.asyncio
 async def test_load_pipeline(mock_app):
     """Test loading a single pipeline"""

tests/unit_tests/pipeline/test_pool.py

@@ -18,6 +18,7 @@ from langbot.pkg.pipeline.pool import (
     ExecutionContextRequiredError,
     QueryNotFoundError,
     QueryPool,
+    QueryPoolCapacityError,
     get_query_execution_context,
 )
 
@@ -433,3 +434,55 @@ class TestQueryPoolWorkspaceIsolation:
         assert pool.get_query_count(workspace_b) == 1
         assert pool.get_query_count(next_generation) == 0
         assert pool.query_id_counter == 3
+
+    async def test_workspace_capacity_discards_oldest_queued_query(self):
+        pool = QueryPool(max_queries=3, max_queries_per_workspace=2)
+        first = await add_scoped_mock_query(pool, TEST_CONTEXT)
+        second = await add_scoped_mock_query(pool, TEST_CONTEXT)
+        third = await add_scoped_mock_query(pool, TEST_CONTEXT)
+
+        assert await pool.get_query(TEST_CONTEXT.workspace_uuid, first.query_uuid) is None
+        assert await pool.get_query(TEST_CONTEXT.workspace_uuid, second.query_uuid) is second
+        assert await pool.get_query(TEST_CONTEXT.workspace_uuid, third.query_uuid) is third
+        assert pool.active_query_count_by_workspace == {TEST_CONTEXT.workspace_uuid: 2}
+        assert pool.get_dropped_query_count(TEST_CONTEXT) == 1
+
+    async def test_capacity_rejects_when_every_query_is_already_running(self):
+        pool = QueryPool(max_queries=1, max_queries_per_workspace=1)
+        running = await add_scoped_mock_query(pool, TEST_CONTEXT)
+        async with pool:
+            pool.mark_query_running_locked(running)
+
+        with pytest.raises(QueryPoolCapacityError):
+            await add_scoped_mock_query(pool, TEST_CONTEXT)
+
+        assert pool.active_query_count_by_workspace == {TEST_CONTEXT.workspace_uuid: 1}
+
+    async def test_mark_query_running_keeps_active_indexes_but_removes_queue_entry(self):
+        pool = QueryPool(max_queries=1, max_queries_per_workspace=1)
+        running = await add_scoped_mock_query(pool, TEST_CONTEXT)
+
+        async with pool:
+            pool.mark_query_running_locked(running)
+
+        assert running not in pool.queries
+        assert await pool.get_query(TEST_CONTEXT.workspace_uuid, running.query_uuid) is running
+        assert pool.active_query_count_by_workspace == {TEST_CONTEXT.workspace_uuid: 1}
+
+    async def test_historical_workspace_counters_are_bounded(self):
+        pool = QueryPool(max_queries=2, max_queries_per_workspace=1)
+        contexts = [
+            ExecutionContext(
+                instance_uuid='instance-test',
+                workspace_uuid=f'workspace-{index}',
+                placement_generation=1,
+            )
+            for index in range(3)
+        ]
+
+        for context in contexts:
+            query = await add_scoped_mock_query(pool, context)
+            await pool.remove_query(query)
+
+        assert len(pool.query_count_by_scope) == 2
+        assert (contexts[0].instance_uuid, contexts[0].workspace_uuid, 1) not in pool.query_count_by_scope

tests/unit_tests/pipeline/test_ratelimit.py

@@ -152,8 +152,18 @@ class TestFixedWindowAlgo:
         # First request creates container
         await algo.require_access(sample_query_with_rate_limit, provider_session.LauncherTypes.PERSON, '12345')
 
-        # Key format: 'LauncherTypes.PERSON_12345' (enum string representation)
-        expected_key = 'LauncherTypes.PERSON_12345'
+        context = sample_query_with_rate_limit._execution_context
+        expected_key = ':'.join(
+            (
+                context.instance_uuid,
+                context.workspace_uuid,
+                str(context.placement_generation),
+                str(sample_query_with_rate_limit.bot_uuid),
+                str(sample_query_with_rate_limit.pipeline_uuid),
+                str(provider_session.LauncherTypes.PERSON),
+                '12345',
+            )
+        )
         assert expected_key in algo.containers
         container = algo.containers[expected_key]
 
@@ -191,8 +201,18 @@ class TestFixedWindowAlgo:
         for i in range(5):
             await algo.require_access(sample_query, provider_session.LauncherTypes.PERSON, 'test')
 
-        # Key format: 'LauncherTypes.PERSON_test'
-        expected_key = 'LauncherTypes.PERSON_test'
+        context = sample_query._execution_context
+        expected_key = ':'.join(
+            (
+                context.instance_uuid,
+                context.workspace_uuid,
+                str(context.placement_generation),
+                str(sample_query.bot_uuid),
+                str(sample_query.pipeline_uuid),
+                str(provider_session.LauncherTypes.PERSON),
+                'test',
+            )
+        )
         container = algo.containers[expected_key]
         assert window_start in container.records
         assert container.records[window_start] == 5