fix(cloud): bound tenant maintenance and monitoring work

tests/unit_tests/api/service/test_maintenance_service.py

@@ -905,6 +905,32 @@ class TestMaintenanceServiceExpiredLocalUploadCandidates:
         # Verify - path included
         assert 'path' in result[0]
 
+    def test_expired_local_upload_candidates_respects_run_limit(self):
+        ap = SimpleNamespace(
+            logger=SimpleNamespace(warning=Mock()),
+            storage_mgr=_scoped_storage_manager(),
+            instance_config=SimpleNamespace(data={'storage': {'cleanup': {'max_files_per_run': 2}}}),
+        )
+        service = MaintenanceService(ap)
+        entries = []
+        for index in range(3):
+            entry = Mock(spec=Path)
+            entry.is_file = Mock(return_value=True)
+            entry.stat = Mock(return_value=SimpleNamespace(st_size=100, st_mtime=0))
+            entry.relative_to = Mock(return_value=Path(f'scoped/old-{index}.txt'))
+            entries.append(entry)
+
+        with patch.object(Path, 'exists', return_value=True):
+            with patch.object(Path, 'rglob', return_value=entries):
+                result = service._expired_local_upload_candidates(TEST_CONTEXT, 7)
+
+        assert [item['key'] for item in result] == [
+            'scoped/old-0.txt',
+            'scoped/old-1.txt',
+        ]
+        ap.instance_config.data['storage']['cleanup']['max_files_per_run'] = 999999
+        assert service._max_files_per_run() == 10000
+
 
 ISOLATION_WORKSPACE_A = '00000000-0000-0000-0000-00000000000a'
 ISOLATION_WORKSPACE_B = '00000000-0000-0000-0000-00000000000b'

tests/unit_tests/api/service/test_monitoring_tenancy.py

@@ -11,7 +11,7 @@ from langbot.pkg.api.http.authz import WorkspaceRequiredError
 from langbot.pkg.api.http.context import ExecutionContext
 from langbot.pkg.api.http.service.monitoring import MonitoringService
 from langbot.pkg.entity.persistence.base import Base
-from langbot.pkg.entity.persistence.monitoring import MonitoringMessage
+from langbot.pkg.entity.persistence.monitoring import MonitoringLLMCall, MonitoringMessage
 from langbot.pkg.entity.persistence.workspace import Workspace
 from langbot.pkg.persistence.mgr import PersistenceManager
 
@@ -176,6 +176,161 @@ async def test_feedback_upsert_and_cancel_are_workspace_scoped(service):
     assert (await service.get_feedback_stats(context_b))['total_feedback'] == 1
 
 
+async def test_monitoring_queries_and_detail_views_are_strictly_bounded(service):
+    context = _context(WORKSPACE_A)
+    service.ap.instance_config.data['monitoring'] = {
+        'query_limits': {
+            'page_rows': 2,
+            'export_rows': 2,
+            'detail_rows': 2,
+            'timeseries_buckets': 2,
+            'max_offset': 10,
+        }
+    }
+    await service.record_session_start(
+        context,
+        session_id='same-session',
+        bot_id='same-bot',
+        bot_name='Same Bot',
+        pipeline_id='same-pipeline',
+        pipeline_name='Same Pipeline',
+    )
+    message_ids = [await _record_message(service, context, f'message-{index}') for index in range(4)]
+    for index in range(3):
+        await service.record_llm_call(
+            context,
+            bot_id='same-bot',
+            bot_name='Same Bot',
+            pipeline_id='same-pipeline',
+            pipeline_name='Same Pipeline',
+            session_id='same-session',
+            model_name='model',
+            input_tokens=1,
+            output_tokens=2,
+            duration=10,
+            message_id=message_ids[0],
+        )
+        await service.record_tool_call(
+            context,
+            tool_name=f'tool-{index}',
+            tool_source='native',
+            duration=5,
+            session_id='same-session',
+            message_id=message_ids[0],
+        )
+        await service.record_error(
+            context,
+            bot_id='same-bot',
+            bot_name='Same Bot',
+            pipeline_id='same-pipeline',
+            pipeline_name='Same Pipeline',
+            error_type='Failure',
+            error_message=f'error-{index}',
+            session_id='same-session',
+            message_id=message_ids[0],
+        )
+
+    page, total = await service.get_messages(context, limit=100000, offset=-5)
+    exported = await service.export_messages(context, limit=100000)
+    session_detail = await service.get_session_analysis(context, 'same-session')
+    message_detail = await service.get_message_details(context, message_ids[0])
+
+    assert total == 4
+    assert len(page) == 2
+    assert len(exported) == 2
+    assert session_detail['message_stats']['total'] == 4
+    assert session_detail['llm_stats']['total_calls'] == 3
+    assert session_detail['tool_stats']['total_calls'] == 3
+    assert len(session_detail['tool_calls']) == 2
+    assert len(session_detail['errors']) == 2
+    assert session_detail['detail_truncated'] == {
+        'tool_calls': True,
+        'errors': True,
+    }
+    assert message_detail['llm_stats']['total_calls'] == 3
+    assert len(message_detail['llm_calls']) == 2
+    assert len(message_detail['errors']) == 2
+    assert message_detail['detail_truncated'] == {
+        'llm_calls': True,
+        'errors': True,
+    }
+
+    service.ap.instance_config.data['monitoring']['query_limits'] = {
+        'page_rows': 999999,
+        'export_rows': 999999,
+        'detail_rows': 999999,
+        'timeseries_buckets': 999999,
+        'max_offset': 99999999,
+    }
+    assert service.normalize_page_window(999999, 99999999) == (5000, 10000000)
+    assert service.normalize_export_limit(999999) == 50000
+    assert service._detail_limit() == 10000
+    assert service._timeseries_bucket_limit() == 10000
+
+
+async def test_token_statistics_aggregate_and_limit_groups_in_database(service):
+    context = _context(WORKSPACE_A)
+    service.ap.instance_config.data['monitoring'] = {
+        'query_limits': {
+            'page_rows': 1,
+            'timeseries_buckets': 2,
+        }
+    }
+    first_hour = datetime.datetime(2026, 7, 28, 10, 0)
+    rows = [
+        {
+            'id': f'llm-{index}',
+            'workspace_uuid': WORKSPACE_A,
+            'timestamp': first_hour + datetime.timedelta(hours=hour, minutes=index),
+            'model_name': model,
+            'input_tokens': input_tokens,
+            'output_tokens': output_tokens,
+            'total_tokens': input_tokens + output_tokens,
+            'duration': 100,
+            'cost': 0.01,
+            'status': 'success',
+            'bot_id': 'same-bot',
+            'bot_name': 'Same Bot',
+            'pipeline_id': 'same-pipeline',
+            'pipeline_name': 'Same Pipeline',
+            'session_id': 'same-session',
+        }
+        for index, (hour, model, input_tokens, output_tokens) in enumerate(
+            [
+                (0, 'small-model', 1, 2),
+                (1, 'large-model', 3, 4),
+                (2, 'large-model', 5, 6),
+                (2, 'large-model', 7, 8),
+            ]
+        )
+    ]
+    await service.ap.persistence_mgr.execute_async(sqlalchemy.insert(MonitoringLLMCall), rows)
+
+    stats = await service.get_token_statistics(context, bucket='hour')
+
+    assert stats['summary']['total_calls'] == 4
+    assert stats['summary']['total_tokens'] == 36
+    assert stats['by_model_truncated'] is True
+    assert [model['model_name'] for model in stats['by_model']] == ['large-model']
+    assert stats['timeseries_truncated'] is True
+    assert stats['timeseries'] == [
+        {
+            'bucket': '2026-07-28 11:00',
+            'input_tokens': 3,
+            'output_tokens': 4,
+            'total_tokens': 7,
+            'calls': 1,
+        },
+        {
+            'bucket': '2026-07-28 12:00',
+            'input_tokens': 12,
+            'output_tokens': 14,
+            'total_tokens': 26,
+            'calls': 2,
+        },
+    ]
+
+
 async def test_cleanup_commits_sqlite_delete_before_vacuum(tmp_path):
     engine = create_async_engine(
         f'sqlite+aiosqlite:///{tmp_path / "monitoring-cleanup.db"}',
@@ -200,32 +355,38 @@ async def test_cleanup_commits_sqlite_delete_before_vacuum(tmp_path):
                 )
             )
             await connection.execute(
-                sqlalchemy.insert(MonitoringMessage).values(
-                    id='expired-message',
-                    workspace_uuid=WORKSPACE_A,
-                    timestamp=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
-                    - datetime.timedelta(days=30),
-                    bot_id='bot',
-                    bot_name='Bot',
-                    pipeline_id='pipeline',
-                    pipeline_name='Pipeline',
-                    message_content='expired',
-                    session_id='session',
-                    status='success',
-                    level='info',
-                )
+                sqlalchemy.insert(MonitoringMessage),
+                [
+                    {
+                        'id': f'expired-message-{index}',
+                        'workspace_uuid': WORKSPACE_A,
+                        'timestamp': datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
+                        - datetime.timedelta(days=30),
+                        'bot_id': 'bot',
+                        'bot_name': 'Bot',
+                        'pipeline_id': 'pipeline',
+                        'pipeline_name': 'Pipeline',
+                        'message_content': 'expired',
+                        'session_id': 'session',
+                        'status': 'success',
+                        'level': 'info',
+                    }
+                    for index in range(5)
+                ],
             )
 
         deleted = await MonitoringService(application).cleanup_expired_records(
             _context(WORKSPACE_A),
             retention_days=1,
+            batch_size=2,
+            max_batches_per_table=1,
         )
 
-        assert deleted['monitoring_messages'] == 1
+        assert deleted['monitoring_messages'] == 2
         async with engine.connect() as connection:
             remaining = await connection.scalar(
                 sqlalchemy.select(sqlalchemy.func.count()).select_from(MonitoringMessage)
             )
-        assert remaining == 0
+        assert remaining == 3
     finally:
         await engine.dispose()

tests/unit_tests/core/test_app_resource_maintenance.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from langbot.pkg.core.app import Application
+
+
+pytestmark = pytest.mark.asyncio
+
+
+class _TaskManager:
+    def __init__(self, stop: asyncio.Event) -> None:
+        self.stop = stop
+        self.tasks: list[asyncio.Task] = []
+
+    def create_task(self, coro, *, name='', **_kwargs):
+        task = asyncio.create_task(coro, name=name)
+        self.tasks.append(task)
+        return SimpleNamespace(task=task)
+
+    async def wait_all(self) -> None:
+        await self.stop.wait()
+        for task in self.tasks:
+            task.cancel()
+        await asyncio.gather(*self.tasks, return_exceptions=True)
+
+
+async def _wait_forever() -> None:
+    await asyncio.Event().wait()
+
+
+async def test_resource_maintenance_waits_and_shares_workspace_discovery() -> None:
+    stop = asyncio.Event()
+    completed = asyncio.Event()
+    discovery_calls = 0
+    job_calls: list[str] = []
+
+    async def list_bindings():
+        nonlocal discovery_calls
+        discovery_calls += 1
+        return [
+            SimpleNamespace(
+                instance_uuid='instance',
+                workspace_uuid='workspace',
+                placement_generation=1,
+            )
+        ]
+
+    async def cleanup_monitoring(_context, _retention_days, *, batch_size):
+        assert batch_size == 10
+        job_calls.append('monitoring')
+        return {}
+
+    async def cleanup_storage(_context):
+        job_calls.append('storage')
+        completed.set()
+        return {}
+
+    application = Application()
+    application.event_loop = asyncio.get_running_loop()
+    application.event_loop_monitor = SimpleNamespace(start=lambda: None)
+    application.task_mgr = _TaskManager(stop)
+    application.plugin_connector = SimpleNamespace(initialize_plugins=lambda: asyncio.sleep(0))
+    application.platform_mgr = SimpleNamespace(run=_wait_forever)
+    application.ctrl = SimpleNamespace(run=_wait_forever)
+    application.http_ctrl = SimpleNamespace(run=_wait_forever)
+    application.telemetry = None
+    application.workspace_collaboration_service = None
+    application.workspace_service = SimpleNamespace(list_active_execution_bindings=list_bindings)
+    application.monitoring_service = SimpleNamespace(cleanup_expired_records=cleanup_monitoring)
+    application.maintenance_service = SimpleNamespace(cleanup_expired_files=cleanup_storage)
+    application.instance_config = SimpleNamespace(
+        data={
+            'monitoring': {
+                'auto_cleanup': {
+                    'enabled': True,
+                    'retention_days': 30,
+                    'delete_batch_size': 10,
+                    'check_interval_hours': 0.00002,
+                }
+            },
+            'storage': {
+                'cleanup': {
+                    'enabled': True,
+                    'check_interval_hours': 0.00002,
+                }
+            },
+        }
+    )
+    application.logger = SimpleNamespace(
+        info=lambda *_args, **_kwargs: None,
+        warning=lambda *_args, **_kwargs: None,
+        error=lambda *_args, **_kwargs: None,
+        debug=lambda *_args, **_kwargs: None,
+    )
+
+    async def no_web_info() -> None:
+        return None
+
+    application.print_web_access_info = no_web_info
+    run_task = asyncio.create_task(application.run())
+    try:
+        await asyncio.sleep(0.01)
+        assert discovery_calls == 0
+        await asyncio.wait_for(completed.wait(), timeout=1)
+        assert discovery_calls == 1
+        assert job_calls == ['monitoring', 'storage']
+    finally:
+        stop.set()
+        await asyncio.wait_for(run_task, timeout=1)

tests/unit_tests/utils/test_httpclient.py

@@ -6,6 +6,7 @@ Tests session management, reuse, and cleanup.
 
 from __future__ import annotations
 
+import asyncio
 import threading
 from types import SimpleNamespace
 from unittest.mock import AsyncMock
@@ -161,14 +162,18 @@ class TestReadLimited:
 
     async def test_httpx_hook_rejects_before_automatic_buffer_grows(self):
         class Source(httpx.AsyncByteStream):
+            def __init__(self):
+                self.closed = False
+
             async def __aiter__(self):
                 yield b'123'
                 yield b'45'
 
             async def aclose(self):
-                return None
+                self.closed = True
 
-        transport = httpx.MockTransport(lambda _request: httpx.Response(200, stream=Source()))
+        source = Source()
+        transport = httpx.MockTransport(lambda _request: httpx.Response(200, stream=source))
         async with httpx.AsyncClient(
             transport=transport,
             event_hooks=httpclient.httpx_response_limit_hooks(max_bytes=4),
@@ -176,6 +181,37 @@ class TestReadLimited:
             with pytest.raises(httpclient.RemoteResponseTooLargeError, match='4-byte'):
                 await client.get('https://example.invalid')
 
+        assert source.closed
+
+    async def test_httpx_limited_stream_closes_source_when_consumer_is_cancelled(self):
+        class Source(httpx.AsyncByteStream):
+            def __init__(self):
+                self.closed = False
+
+            async def __aiter__(self):
+                yield b'123'
+                await asyncio.Event().wait()
+
+            async def aclose(self):
+                self.closed = True
+
+        source = Source()
+        stream = httpclient._LimitedHTTPXAsyncByteStream(source, max_bytes=4)
+        first_chunk_consumed = asyncio.Event()
+
+        async def consume():
+            async for _chunk in stream:
+                first_chunk_consumed.set()
+
+        task = asyncio.create_task(consume())
+        await first_chunk_consumed.wait()
+        task.cancel()
+
+        with pytest.raises(asyncio.CancelledError):
+            await task
+
+        assert source.closed
+
     async def test_close_all_handles_already_closed(self):
         """close_all handles already closed sessions gracefully."""
         session = httpclient.get_session()